Merge pull request #13 from xlang-ai/xiaochuanli/addChromeExtensions
Xiaochuanli/add chrome extensions
This commit is contained in:
Tianbao Xie
@@ -24,6 +24,7 @@ from .chrome import (
|
||||
get_gotoRecreationPage_and_get_html_content,
|
||||
get_url_dashPart,
|
||||
get_active_url_from_accessTree,
|
||||
get_find_installed_extension_name,
|
||||
get_info_from_website
|
||||
)
|
||||
from .file import get_cloud_file, get_vm_file, get_cache_file, get_content_from_vm_file
|
||||
@@ -31,7 +32,8 @@ from .general import get_vm_command_line, get_vm_terminal_output, get_vm_command
|
||||
from .gimp import get_gimp_config_file
|
||||
from .impress import get_audio_in_slide, get_background_image_in_slide
|
||||
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
|
||||
from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime
|
||||
from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime, get_time_diff_range
|
||||
from .replay import get_replay
|
||||
from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player
|
||||
from .vscode import get_vscode_config
|
||||
from .calc import get_conference_city_in_order
|
||||
15
desktop_env/evaluators/getters/calc.py
Normal file
15
desktop_env/evaluators/getters/calc.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import csv
|
||||
|
||||
# I want to write a function, reads a csv file, and get all the contents in the third column in the order of rows
|
||||
def get_conference_city_in_order(env, config):
|
||||
# read the csv file
|
||||
csv_path = config['csv_path']
|
||||
print(f"Reading csv file from {csv_path}")
|
||||
with open(csv_path, 'r') as f:
|
||||
reader = csv.reader(f)
|
||||
# skip the header row
|
||||
next(reader)
|
||||
# get the third column in the order of rows
|
||||
conference_city_list = [row[2] for row in reader]
|
||||
return conference_city_list
|
||||
|
||||
@@ -4,6 +4,7 @@ import os
|
||||
import platform
|
||||
import sqlite3
|
||||
import time
|
||||
from urllib.parse import unquote
|
||||
from typing import Dict, Any, List
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
@@ -1010,6 +1011,43 @@ def get_find_unpacked_extension_path(env, config: Dict[str, str]):
|
||||
return "Google"
|
||||
|
||||
|
||||
def get_find_installed_extension_name(env, config: Dict[str, str]):
|
||||
os_type = env.vm_platform
|
||||
if os_type == 'Windows':
|
||||
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
|
||||
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
|
||||
elif os_type == 'Darwin':
|
||||
preference_file_path = env.controller.execute_python_command(
|
||||
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
|
||||
'output'].strip()
|
||||
elif os_type == 'Linux':
|
||||
if "arm" in platform.machine():
|
||||
preference_file_path = env.controller.execute_python_command(
|
||||
"import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[
|
||||
'output'].strip()
|
||||
else:
|
||||
preference_file_path = env.controller.execute_python_command(
|
||||
"import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
|
||||
'output'].strip()
|
||||
|
||||
else:
|
||||
raise Exception('Unsupported operating system')
|
||||
|
||||
try:
|
||||
content = env.controller.get_file(preference_file_path)
|
||||
data = json.loads(content)
|
||||
# Preferences store all the path of installed extensions, return them all and let metrics try to find one matches the targeted extension path
|
||||
all_extensions_name = []
|
||||
all_extensions = data.get('extensions', {}).get('settings', {})
|
||||
for id in all_extensions.keys():
|
||||
name = all_extensions[id]["manifest"]["name"]
|
||||
all_extensions_name.append(name)
|
||||
return all_extensions_name
|
||||
except Exception as e:
|
||||
logger.error(f"Error: {e}")
|
||||
return "Google"
|
||||
|
||||
|
||||
def get_data_delete_automacally(env, config: Dict[str, str]):
|
||||
"""
|
||||
This function is used to open th "auto-delete" mode of chromium
|
||||
@@ -1077,6 +1115,7 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]):
|
||||
"""
|
||||
active_tab_url = get_active_url_from_accessTree(env, config)
|
||||
if not isinstance(active_tab_url, str):
|
||||
logger.error("active_tab_url is not a string")
|
||||
return None
|
||||
host = env.vm_ip
|
||||
port = 9222 # fixme: this port is hard-coded, need to be changed from config file
|
||||
@@ -1109,12 +1148,14 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]):
|
||||
for context in browser.contexts:
|
||||
for page in context.pages:
|
||||
page.wait_for_load_state("networkidle")
|
||||
if page.url == active_tab_url:
|
||||
# the accTree and playwright can get encoding(percent-encoding) characters, we need to convert them to normal characters
|
||||
if unquote(page.url) == unquote(active_tab_url):
|
||||
target_page = page
|
||||
print("tartget page url: ", target_page.url)
|
||||
print("tartget page title: ", target_page.title())
|
||||
print("\33[32mtartget page url: ", target_page.url, "\33[0m")
|
||||
print("\33[32mtartget page title: ", target_page.title(), "\33[0m")
|
||||
break
|
||||
if target_page is None:
|
||||
logger.error("Your tab is not the target tab.")
|
||||
return {}
|
||||
return_json = {}
|
||||
if config["category"] == "class":
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
from typing import Dict, List, Set
|
||||
from typing import Optional, Any, Union
|
||||
from datetime import datetime
|
||||
import requests
|
||||
import pandas as pd
|
||||
|
||||
@@ -77,14 +78,24 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option
|
||||
gives (List[int]): optional. defaults to [0]. which files are directly
|
||||
returned to the metric. if len==1, str is returned; else, list is
|
||||
returned.
|
||||
only support for single file now:
|
||||
time_suffix(bool): optional. defaults to False. if True, append the current time in required format.
|
||||
time_format(str): optional. defaults to "%Y_%m_%d". format of the time suffix.
|
||||
"""
|
||||
|
||||
time_format = "%Y_%m_%d"
|
||||
if not config.get("multi", False):
|
||||
paths: List[str] = [config["path"]]
|
||||
dests: List[str] = [config["dest"]]
|
||||
else:
|
||||
paths: List[str] = config["path"]
|
||||
dests: List[str] = config["dest"]
|
||||
if "time_suffix" in config.keys() and config["time_suffix"]:
|
||||
if "time_format" in config.keys():
|
||||
time_format = config["time_format"]
|
||||
# Insert time before . in file type suffix
|
||||
paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths]
|
||||
dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests]
|
||||
|
||||
cache_paths: List[str] = []
|
||||
|
||||
gives: Set[int] = set(config.get("gives", [0]))
|
||||
@@ -104,7 +115,8 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option
|
||||
cache_paths.append(_path)
|
||||
with open(_path, "wb") as f:
|
||||
f.write(file)
|
||||
|
||||
# debug
|
||||
print(cache_paths)
|
||||
return cache_paths[0] if len(cache_paths)==1 else cache_paths
|
||||
|
||||
|
||||
|
||||
@@ -195,3 +195,10 @@ def get_accessibility_tree(env, *args) -> str:
|
||||
accessibility_tree: str = env.controller.get_accessibility_tree()
|
||||
logger.debug("AT@eval: %s", accessibility_tree)
|
||||
return accessibility_tree
|
||||
|
||||
def get_time_diff_range(env, config) -> str:
|
||||
try:
|
||||
return config["diff_range_in_minutes"]
|
||||
except:
|
||||
logger.error("diff_range_in_minutes not found in config.")
|
||||
return None
|
||||
@@ -2,7 +2,8 @@ from .basic_os import (
|
||||
check_gnome_favorite_apps,
|
||||
is_utc_0,
|
||||
check_text_enlarged,
|
||||
check_moved_jpgs
|
||||
check_moved_jpgs,
|
||||
is_in_vm_clickboard
|
||||
)
|
||||
from .chrome import (
|
||||
is_expected_tabs,
|
||||
@@ -19,6 +20,7 @@ from .chrome import (
|
||||
is_expected_active_tab,
|
||||
is_expected_url_pattern_match,
|
||||
is_added_to_steam_cart,
|
||||
is_expected_installed_extensions
|
||||
compare_pdf_images
|
||||
)
|
||||
from .docs import (
|
||||
@@ -47,6 +49,7 @@ from .docs import (
|
||||
check_file_exists,
|
||||
check_tabstops,
|
||||
compare_contains_image,
|
||||
compare_docx_files_and_ignore_new_lines
|
||||
compare_docx_images,
|
||||
compare_image_text,
|
||||
compare_references
|
||||
@@ -62,6 +65,14 @@ from .general import (
|
||||
fuzzy_match,
|
||||
check_include_exclude,
|
||||
check_direct_json_object,
|
||||
compare_time_in_speedtest_results,
|
||||
is_included_all_json_objects,
|
||||
is_gold_text_included_in_pdf,
|
||||
check_csv_line_number,
|
||||
file_contains,
|
||||
compare_terminal_and_txt,
|
||||
fuzzy_place_math,
|
||||
compare_python_pure_text
|
||||
diff_text_file,
|
||||
literal_match
|
||||
)
|
||||
@@ -137,6 +148,7 @@ from .vscode import (
|
||||
check_html_background_image,
|
||||
compare_zip_files
|
||||
)
|
||||
from .calc import compare_conference_city_in_order
|
||||
from .others import compare_epub, check_mp3_meta
|
||||
|
||||
def infeasible():
|
||||
|
||||
@@ -56,3 +56,15 @@ def check_moved_jpgs(directory_list, rule):
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
def is_in_vm_clickboard(config, terminal_output):
|
||||
print("terminal_output: ")
|
||||
print(terminal_output)
|
||||
print("config: ")
|
||||
print(config)
|
||||
expected_results = config["expected"]
|
||||
# check if terminal_output has expected results
|
||||
if not isinstance(expected_results, list):
|
||||
return 1 if expected_results in terminal_output else 0
|
||||
else:
|
||||
return 1 if all(result in terminal_output for result in expected_results) else 0
|
||||
24
desktop_env/evaluators/metrics/calc.py
Normal file
24
desktop_env/evaluators/metrics/calc.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import openpyxl
|
||||
|
||||
def compare_conference_city_in_order( actual_city_list_path, expected_city):
|
||||
expected_city_list = expected_city["expected"]
|
||||
print(f"Reading csv file from {actual_city_list_path}")
|
||||
wb = openpyxl.load_workbook(actual_city_list_path)
|
||||
sheet = wb.active
|
||||
actual_city_list = []
|
||||
for row in sheet["C2:C22"]:
|
||||
for cell in row:
|
||||
actual_city_list.append(cell.value)
|
||||
# expected_city is the city that we want to compare with the actual city list
|
||||
# must in order index
|
||||
# debug
|
||||
print("expected_city_list:")
|
||||
print(expected_city_list)
|
||||
print("actual_city_list_path:")
|
||||
print(actual_city_list)
|
||||
wrong_list = []
|
||||
for i in range(len(actual_city_list)):
|
||||
if expected_city_list[i] not in actual_city_list[i]:
|
||||
wrong_list.append(i)
|
||||
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
|
||||
return True if len(wrong_list) == 0 else False
|
||||
@@ -61,6 +61,12 @@ def is_expected_url_pattern_match(result, rules) -> float:
|
||||
return 1.
|
||||
|
||||
|
||||
def is_expected_installed_extensions(installed_extensions, expected) -> float:
|
||||
print("installed_extensions: ")
|
||||
print(installed_extensions)
|
||||
expected_extensions = expected["expected"]
|
||||
return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed
|
||||
|
||||
def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float:
|
||||
"""
|
||||
Checks if the expected tabs are open in Chrome.
|
||||
|
||||
28
desktop_env/evaluators/metrics/demo.py
Normal file
28
desktop_env/evaluators/metrics/demo.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import fitz # PyMuPDF
|
||||
|
||||
def extract_answers_from_pdf(pdf_file):
|
||||
# 打开PDF文件
|
||||
doc = fitz.open(pdf_file)
|
||||
answers = []
|
||||
|
||||
# 遍历每一页
|
||||
for page in doc:
|
||||
# 提取当前页的文本
|
||||
text = page.get_text()
|
||||
# 分割文本为行
|
||||
lines = text.split('\n')
|
||||
for line in lines:
|
||||
if line.strip(): # 排除空白行
|
||||
# 分割等号，提取答案
|
||||
parts = line.split('=')
|
||||
if len(parts) > 1:
|
||||
answer = parts[-1].strip() # 取等号后的部分为答案
|
||||
answers.append(answer)
|
||||
|
||||
return answers
|
||||
|
||||
# 假设你的文件名是'math_problems.pdf'
|
||||
pdf_file = '/Users/lxc/Desktop/calculus.pdf'
|
||||
answers = extract_answers_from_pdf(pdf_file)
|
||||
for i, answer in enumerate(answers, 1):
|
||||
print(f"题目{i}的答案是: {answer}")
|
||||
@@ -118,14 +118,21 @@ def compare_docx_files(file1, file2, **options):
|
||||
if text1 != text2:
|
||||
return 0
|
||||
else:
|
||||
print("ignore_blanks=false")
|
||||
if len(doc1_paragraphs) != len(doc2_paragraphs):
|
||||
print(doc1_paragraphs)
|
||||
print(doc2_paragraphs)
|
||||
print(len(doc1_paragraphs))
|
||||
print(len(doc2_paragraphs))
|
||||
return 0
|
||||
|
||||
print("in compare")
|
||||
# Compare each paragraph
|
||||
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
|
||||
if ignore_case:
|
||||
p1, p2 = p1.lower(), p2.lower()
|
||||
if p1 != p2:
|
||||
print(p1)
|
||||
print(p2)
|
||||
return 0
|
||||
|
||||
return 1
|
||||
@@ -552,6 +559,39 @@ def compare_docx_lines(file1, file2):
|
||||
return 0
|
||||
|
||||
|
||||
def compare_docx_files_and_ignore_new_lines(file1, file2, **options):
|
||||
ignore_blanks = options.get('ignore_blanks', True)
|
||||
|
||||
# Determine file types and load documents
|
||||
if file1.endswith('.docx') and file2.endswith('.docx'):
|
||||
doc1 = Document(file1)
|
||||
doc2 = Document(file2)
|
||||
# First, delete all the blank in paragraphs
|
||||
doc1 = [p for p in doc1.paragraphs if p.text != '']
|
||||
doc2 = [p for p in doc2.paragraphs if p.text != '']
|
||||
doc1_paragraphs = [p.text for p in doc1]
|
||||
doc2_paragraphs = [p.text for p in doc2]
|
||||
else:
|
||||
# Unsupported file types or mismatch
|
||||
print("Unsupported file types or mismatch between file types.")
|
||||
return 0
|
||||
|
||||
# Process and compare documents
|
||||
if ignore_blanks:
|
||||
text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
|
||||
text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
|
||||
if text1 != text2:
|
||||
return 0
|
||||
else:
|
||||
if len(doc1_paragraphs) != len(doc2_paragraphs):
|
||||
return 0
|
||||
# Compare each paragraph
|
||||
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
|
||||
if p1 != p2:
|
||||
return 0
|
||||
return 1
|
||||
|
||||
|
||||
# Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated
|
||||
def compare_highlighted_text(file1, file2):
|
||||
def extract_highlighted_text(file_path):
|
||||
|
||||
@@ -4,15 +4,18 @@ import json
|
||||
import yaml
|
||||
import operator
|
||||
import re
|
||||
import pdfplumber
|
||||
import sqlite3
|
||||
from numbers import Number
|
||||
from typing import Callable, Any, Union
|
||||
from typing import Dict, List, Pattern
|
||||
|
||||
import datetime
|
||||
import pandas as pd
|
||||
import lxml.etree
|
||||
from lxml.cssselect import CSSSelector
|
||||
from lxml.etree import _Element
|
||||
from rapidfuzz import fuzz
|
||||
from docx import Document
|
||||
import difflib
|
||||
|
||||
from .utils import _match_record, _match_value_to_rule
|
||||
@@ -84,6 +87,21 @@ def fuzzy_match(result, rules) -> float:
|
||||
return fuzz.ratio(result, expect) / 100.
|
||||
|
||||
|
||||
def fuzzy_place_math(result_file_path, rules) -> float:
|
||||
expect = rules["expected"] # a list of possible answers
|
||||
# read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line
|
||||
doc = Document(result_file_path)
|
||||
words_list = []
|
||||
for para in doc.paragraphs:
|
||||
words_list.extend(para.text.split())
|
||||
# Print out the list of extracted words
|
||||
print(words_list)
|
||||
for word in words_list:
|
||||
if not any(ans in word for ans in expect):
|
||||
print("Wrong place:", word)
|
||||
return 0
|
||||
return 1
|
||||
|
||||
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
|
||||
"""
|
||||
Args:
|
||||
@@ -272,15 +290,137 @@ def check_direct_json_object(result, rules)->float:
|
||||
One of the most commonly used function to evalute.
|
||||
Compare two json objects directly.
|
||||
"""
|
||||
if isinstance(result, str):
|
||||
# remove blanks before and after result
|
||||
result = result.strip()
|
||||
# replace all ' with "
|
||||
result = result.replace("'", '"')
|
||||
# load json object
|
||||
result = json.loads(result)
|
||||
print("result: ")
|
||||
print(result)
|
||||
print("expected: ")
|
||||
print(rules["expected"])
|
||||
if result is None:
|
||||
return 0.
|
||||
expected_json = rules["expected"]
|
||||
for key in expected_json.keys():
|
||||
expected_value = expected_json.get(key)
|
||||
if expected_value != result.get(key):
|
||||
return 0.
|
||||
return 1.0
|
||||
|
||||
expect_in_result = rules.get("expect_in_result", False)
|
||||
if not expect_in_result:
|
||||
expected_json = rules["expected"]
|
||||
for key in expected_json.keys():
|
||||
expected_value = expected_json.get(key)
|
||||
if expected_value != result.get(key):
|
||||
return 0.
|
||||
return 1.0
|
||||
else:
|
||||
expected_json = rules["expected"]
|
||||
for key in expected_json.keys():
|
||||
expected_value = expected_json.get(key)
|
||||
if expected_value not in result.get(key):
|
||||
return 0.
|
||||
return 1.0
|
||||
|
||||
def compare_time_in_speedtest_results(speedtest_result_path, time_diff):
|
||||
# open the speedtest results file(csv)
|
||||
date_col = None
|
||||
with open(speedtest_result_path, 'r') as f:
|
||||
reader = pd.read_csv(f)
|
||||
for column in reader.columns:
|
||||
if column.startswith('TEST_DATE'):
|
||||
date_col = column
|
||||
break
|
||||
now_date_time = datetime.datetime.now().strftime('%H:%M')
|
||||
for date in reader[date_col]:
|
||||
date_time = date[-5:]
|
||||
# compare the date time with the current date time, if time diff less than time_diff para, then return true
|
||||
if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, '%H:%M')).total_seconds()) / 60 < int(time_diff):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_included_all_json_objects(gold_file_path, result_file_path):
|
||||
print("gold_file_path: ")
|
||||
print(gold_file_path)
|
||||
print("result_file_path: ")
|
||||
print(result_file_path)
|
||||
# two json file, check if all the key-value pair in gold_file_path is included in result_file_path
|
||||
with open(gold_file_path, 'r') as f:
|
||||
gold_json = json.load(f)
|
||||
with open(result_file_path, 'r') as fr:
|
||||
result_json = json.load(fr)
|
||||
for key in gold_json.keys():
|
||||
if key not in result_json.keys() or gold_json[key] != result_json[key]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path):
|
||||
print("gold_text_path: ")
|
||||
print(gold_text_path)
|
||||
print("pdf_file_path: ")
|
||||
print(pdf_file_path)
|
||||
# gold file is a json file, we need to check all the value in json are included in pdf file.
|
||||
with open(gold_text_path, 'r') as f:
|
||||
gold_json = json.load(f)
|
||||
with pdfplumber.open(pdf_file_path) as pdf:
|
||||
text = ''
|
||||
for page in pdf.pages:
|
||||
text += page.extract_text()
|
||||
false_list = []
|
||||
for key in gold_json.keys():
|
||||
if gold_json[key] not in text:
|
||||
false_list.append(key)
|
||||
if len(false_list) > 0:
|
||||
print("false_list: ")
|
||||
print(false_list)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def file_contains(file_path, config):
|
||||
# file_path ends with .txt
|
||||
if not file_path :
|
||||
return False
|
||||
with open(file_path, 'r') as f:
|
||||
file_text = f.read()
|
||||
for text in config["expected"]:
|
||||
if text not in file_text:
|
||||
return False
|
||||
return True
|
||||
|
||||
def check_csv_line_number(file_path, line_number):
|
||||
# check file_path suffix
|
||||
if not file_path.endswith('.csv'):
|
||||
return False
|
||||
# check line number
|
||||
with open(file_path, 'r') as f:
|
||||
reader = csv.reader(f)
|
||||
line_count = sum(1 for row in reader)
|
||||
return True if line_count == int(line_number["expected"]) else False
|
||||
|
||||
|
||||
def compare_terminal_and_txt(txt_file_path, terminal_output):
|
||||
# read txt file content
|
||||
with open(txt_file_path, 'r') as f:
|
||||
txt_file_content = f.read()
|
||||
# compare terminal output with txt file content
|
||||
return True if terminal_output == txt_file_content else False
|
||||
|
||||
|
||||
def compare_python_pure_text(py_file_path, gold_file_path):
|
||||
# first, change the suffix of gold_file from .txt to .py
|
||||
print("py_file_path: ")
|
||||
print(py_file_path)
|
||||
print("gold_file_path: ")
|
||||
print(gold_file_path)
|
||||
# gold_file_path = gold_file_path.replace('.txt', '.py')
|
||||
def remove_whitespace(text):
|
||||
return ''.join(text.split())
|
||||
with open(py_file_path, 'r') as file1:
|
||||
content1 = file1.read()
|
||||
with open(gold_file_path, 'r') as file2:
|
||||
content2 = file2.read()
|
||||
content1_no_whitespace = remove_whitespace(content1)
|
||||
content2_no_whitespace = remove_whitespace(content2)
|
||||
return content1_no_whitespace == content2_no_whitespace
|
||||
@@ -1,10 +1,12 @@
|
||||
import builtins
|
||||
import datetime
|
||||
import functools
|
||||
import itertools
|
||||
import logging
|
||||
import operator
|
||||
import re
|
||||
import zipfile
|
||||
import pandas as pd
|
||||
from typing import Any, TypeVar, Union, Iterable, Optional, Callable
|
||||
from typing import Dict, List, Set, Match, Tuple, Pattern
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
@@ -53,17 +53,32 @@
|
||||
"chrome"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "is_expected_active_tab",
|
||||
"result": {
|
||||
"func": ["is_expected_active_tab", "is_expected_active_tab"],
|
||||
"conj": "or",
|
||||
"result": [
|
||||
{
|
||||
"type": "active_url_from_accessTree",
|
||||
"goto_prefix": "https://www."
|
||||
},
|
||||
"expected": {
|
||||
},
|
||||
{
|
||||
"type": "active_url_from_accessTree",
|
||||
"goto_prefix": "https://www."
|
||||
}
|
||||
],
|
||||
"expected": [
|
||||
{
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"type": "url",
|
||||
"url": "https://www.drugs.com/npc/"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"type": "url",
|
||||
"url": "https://www.drugs.com/npp/"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,8 +62,8 @@
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected": {
|
||||
"locationName": "Zurich Airport",
|
||||
"dropLocationName": "Zurich Airport",
|
||||
"locationName": "Zürich",
|
||||
"dropLocationName": "Zürich",
|
||||
"filterCriteria_carCategory": "large",
|
||||
"filterCriteria_sortBy": "PRICE"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
|
||||
"snapshot": "chrome",
|
||||
"instruction": "Can you make a new folder for me on that bookmarks bar in my internet browser? Let's call it 'Favorites.'",
|
||||
"instruction": "Can you make a new folder for me on the bookmarks bar in my internet browser? Let's call it 'Favorites.'",
|
||||
"source": "https://www.youtube.com/watch?v=IN-Eq_UripQ",
|
||||
"config": [
|
||||
{
|
||||
|
||||
@@ -54,7 +54,7 @@
|
||||
"rules":{
|
||||
"expected": {
|
||||
"q": "drip coffee maker",
|
||||
"tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,pdtr0:1825161|1825162"
|
||||
"tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,sales:1,pdtr0:1825161|1825162"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,19 +43,35 @@
|
||||
"chrome"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "exact_match",
|
||||
"result": {
|
||||
"func": ["exact_match", "exact_match"],
|
||||
"conj": "or",
|
||||
"result": [
|
||||
{
|
||||
"type": "url_dashPart",
|
||||
"goto_prefix": "https://www.",
|
||||
"partIndex": -1,
|
||||
"needDeleteId": false,
|
||||
"returnType": "string"
|
||||
},
|
||||
"expected": {
|
||||
},
|
||||
{
|
||||
"type": "url_dashPart",
|
||||
"goto_prefix": "https://www.",
|
||||
"partIndex": -1,
|
||||
"needDeleteId": false,
|
||||
"returnType": "string"
|
||||
}],
|
||||
"expected": [
|
||||
{
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"expected": "tamiflu.html#side-effects"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"expected": "tamiflu-side-effects.html"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@
|
||||
"rules": {
|
||||
"expected": [
|
||||
"AgeAppropriate:Kids",
|
||||
"search=spider-man%20toys",
|
||||
"search=spider[-%20]?man%20toys",
|
||||
"S=4"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"id": "2373b66a-092d-44cb-bfd7-82e86e7a3b4d",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor the system CPU usage, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop, and convert the txt to csv file with the same name.",
|
||||
"source": "author",
|
||||
"config": [
|
||||
{
|
||||
"type": "command",
|
||||
"parameters":{
|
||||
"command": "echo lixiaochuan20 | sudo -S apt-get update && echo lixiaochuan20 | sudo -S apt-get install sysstat",
|
||||
"shell": "true"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"os", "calc"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": ["file_contains", "check_csv_line_number"],
|
||||
"result":
|
||||
[
|
||||
{
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/System_Resources_Report.txt",
|
||||
"dest": "System_Resources_Report.txt"
|
||||
},
|
||||
{
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/System_Resources_Report.csv",
|
||||
"dest": "System_Resources_Report.csv"
|
||||
}
|
||||
],
|
||||
"expected":
|
||||
[
|
||||
{
|
||||
"type": "rule",
|
||||
"rules" :{
|
||||
"expected": ["tps", "rtps", "wtps", "dtps", "bread/s", "bwrtn/s", "bdscd/s", "Average", "Linux"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"expected": "33"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,69 @@
|
||||
{
|
||||
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
|
||||
"snapshot": "libreoffice_calc",
|
||||
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
],
|
||||
"trajectory": "trajectories/26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
|
||||
"related_apps": [
|
||||
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig": [],
|
||||
"func": "",
|
||||
"result": {
|
||||
|
||||
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed (if the dir does not exist, create it).",
|
||||
"source": "https://www.speedtest.net/",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://www.speedtest.net/"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "Google Chrome"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"python",
|
||||
"-c",
|
||||
"import pyautogui; import time; time.sleep(0.5);"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps":[
|
||||
"os",
|
||||
"browser"
|
||||
],
|
||||
"evaluator":{
|
||||
"func": "compare_time_in_speedtest_results",
|
||||
"result":{
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Test/Speed/Speedtest Results Export-.csv",
|
||||
"dest": "Speedtest Results Export-.csv",
|
||||
"time_suffix": true
|
||||
},
|
||||
"expected":{
|
||||
"type": "time_diff_range",
|
||||
"diff_range_in_minutes": "60"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"id": "3eb2a122-a5e3-4f89-9820-f7fa1a582969",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "Please search online for the submission deadline and venue of the ICLR main conference in 2035, and copy it to my clipboard. If not yet publicized, copy None.",
|
||||
"source": "author",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"parameters":{
|
||||
"command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc",
|
||||
"shell": "true"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"os", "chrome"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "is_in_vm_clickboard",
|
||||
"expected": {
|
||||
"type": "vm_command_line",
|
||||
"command": "xsel --clipboard --output",
|
||||
"shell": "true"
|
||||
},
|
||||
"result": {
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"expected": ["None"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"id": "5bc63fb9-276a-4439-a7c1-9dc76401737f",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I have a JSON-formatted data file opened now that stores the responses of several large language models on a specific dataset. Now, I want to filter out all the responses from Gemini and specifically look at the sentences in the responses that contain \"Iliad\". Please copy all of Gemini's responses(values) from the JSON file, paste them into a Word document named \"gemini_results.docx\" under Desktop. Each response should be a new paragraph and sepreated by a new line. Highlight the all the \"Iliad\" word.",
|
||||
"source": "",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6",
|
||||
"path": "/home/user/Desktop/llm_answers.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/Desktop/llm_answers.json"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"libreoffice_writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig": [
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "gemini_results.docx - LibreOffice Writer",
|
||||
"strict": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sleep",
|
||||
"parameters": {
|
||||
"seconds": 0.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"python",
|
||||
"-c",
|
||||
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "compare_docx_files_and_ignore_new_lines",
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.google.com/uc?export=download&id=1N5LlYhrRWtcC24LKiDQAYJYqCDYMBh9b",
|
||||
"dest": "gemini_results_Gold.docx"
|
||||
},
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/gemini_results.docx",
|
||||
"dest": "gemini_results.docx"
|
||||
},
|
||||
"options": {
|
||||
"ignore_blanks": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"id": "65e62ec6-4603-4c67-b4d2-07830deb285b",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I've recently become very interested in large language models. I tried entering some questions into several large language models and stored the answers in a file named \"answers.json\" under Desktop. Now, I want to focus only on the answers from GPT-4. Please help me copy all the GPT-4 answers into a Word document named \"results.docx\" under Desktop, one answer per paragraph.",
|
||||
"source": "",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6",
|
||||
"path": "/home/user/Desktop/answers.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/Desktop/answers.json"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"libreoffice_writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig": [
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "results.docx - LibreOffice Writer",
|
||||
"strict": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sleep",
|
||||
"parameters": {
|
||||
"seconds": 0.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"python",
|
||||
"-c",
|
||||
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "compare_docx_files_and_ignore_new_lines",
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.google.com/uc?export=download&id=1edtU4kHecg7oxerf1jvd1-ak-2WmEla3",
|
||||
"dest": "results_Gold.docx"
|
||||
},
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/results.docx",
|
||||
"dest": "results.docx"
|
||||
},
|
||||
"options": {
|
||||
"ignore_blanks": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.",
|
||||
"source": "author",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1MhoRWCk-_ZZYg1agfQWHaOEYxXOrXLKC",
|
||||
"path": "/home/user/Desktop/ConferenceCity.xlsx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1p10oM6S3iZ6x2UVCgZE1mQ9D1yeZGCbO",
|
||||
"path": "/home/user/Desktop/ConferenceCity_Gold.xlsx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/Desktop/ConferenceCity.xlsx"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"calc", "chrome", "os"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "compare_conference_city_in_order",
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"]
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/ConferenceCity.xlsx",
|
||||
"dest": "ConferenceCity.xlsx"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"id": "716a6079-22da-47f1-ba73-c9d58f986a38",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I remember there is a file named \"secret.docx\" on this computer, but I can't remember where it is. Please find the path where this file is stored and copy it to the clipboard.",
|
||||
"source": "",
|
||||
"config": [
|
||||
{
|
||||
"type": "command",
|
||||
"parameters": {
|
||||
"command": "mkdir -p /home/user/Data1/List1 && mkdir -p /home/user/Data2/List2 && mkdir -p /home/user/Data3/List3 && mkdir -p /home/user/Data4/List4 && mkdir -p /home/user/Data5/List5",
|
||||
"shell" :"true"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1kjfHFNmAy4ZVqGgFsxPDxWwMuZkCHxAv",
|
||||
"path": "/home/user/Data3/List3/secret.docx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"parameters":{
|
||||
"command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc",
|
||||
"shell": "true"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"os", "terminal"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "is_in_vm_clickboard",
|
||||
"expected": {
|
||||
"type": "vm_command_line",
|
||||
"command": "xsel --clipboard --output",
|
||||
"shell": "true"
|
||||
},
|
||||
"result": {
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"expected": "/home/user/Data3/List3/secret.docx"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"id": "7ff48d5b-2df2-49da-b500-a5150ffc7f18",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I am a Chinese citizen and I want to go to Macau to watch a concert recently, but I have not yet applied for a visa for Macau. I live in Futian District, Shenzhen City. I heard that Shenzhen currently has 24-hour self-service check-in machines. Please help me find the addresses of 5 24-hour self-service check-in machines in Futian District and save them in Chinese in this open word document.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1wrxlOrxZr3I5vaF1KRzc9nA9UcWW2ReU",
|
||||
"path": "/home/user/Desktop/AllLocations.docx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/Desktop/AllLocations.docx"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "AllLocations.docx - LibreOffice Writer"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome", "os", "writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "fuzzy_place_math",
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/AllLocations.docx",
|
||||
"dest": "AllLocations.docx"
|
||||
},
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected": ["正义", "海天综合大厦", "盛世家园", "香梅路", "金沙嘴大厦", "卓悦汇", "昌泰公寓", "天安数码城", "振兴路", "梅岗南街", "滨河大道", "上沙", "香蜜原著", "八卦岭", "竹园", "竹子林五路", "体育中心", "赤尾", "书城中心广场", "侨城东路", "沙头所", "水围村", "黄木岗", "华强广场", "国大药房","科学馆","江苏宾馆", "梅林"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"id": "873cafdd-a581-47f6-8b33-b9696ddb7b05",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "My friend is a \"plugin guru\" and he recommended some good plug-ins to me. Please go to the Chrome plug-in store and install all the listed plug-ins.",
|
||||
"source": "author",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1el4WdiJVpUP49FABN_DpC7yy7obgJSC5",
|
||||
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "Recommended_plugin_list.docx - LibreOffice Writer"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"os", "chrome", "Writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "is_expected_installed_extensions",
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"]
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"type": "find_installed_extension_name"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
{
|
||||
"id": "9f3bb592-209d-43bc-bb47-d77d9df56504",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I downloaded a video to practice listening, but I don't know how to remove the subtitles. Please help me remove the subtitles from the video and export it as \"subtitles.srt\" and store it in the same directory as the video.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1YrPG1sXDACdXlrBlW8-RUzovWUJ0qoC5",
|
||||
"path": "/home/user/video.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters":{
|
||||
"command":"echo lixiaochuan20 | sudo -S mv /home/user/video.txt /home/user/video.mp4",
|
||||
"shell": "true"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/video.mp4"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "video.mp4 - VLC media player"
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"os", "vlc"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig":[
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1Q6GV39h74bc_8_kFKjNW3RMxUSgXaaWR",
|
||||
"path": "/home/user/subtitles_Gold.srt"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=14eBaLVfUekkpj3dPkdDwHehHRmzx2njy",
|
||||
"path": "/home/user/Desktop/subtitles_script.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"parameters": {
|
||||
"command": "echo lixiaochuan20 | sudo -S pip install pysrt",
|
||||
"shell" :"true"
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "exact_match",
|
||||
"result": {
|
||||
"type": "vm_command_line",
|
||||
"command": "echo lixiaochuan20 | mv /home/user/Desktop/subtitles_script.txt /home/user/Desktop/subtitles_script.py && python /home/user/Desktop/subtitles_script.py",
|
||||
"shell" :"true"
|
||||
},
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected": "true\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"id": "a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I have developed a new Chrome extension myself, so it needs to be installed manually. Please help me install the extension located in the Desktop directory into the Chrome browser.",
|
||||
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files":[
|
||||
{
|
||||
"url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download",
|
||||
"path":"/home/user/Desktop/helloExtension.zip"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": "echo password | sudo -S apt-get update -y && echo password | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip",
|
||||
"shell": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome", "os"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "is_in_list",
|
||||
"result": {
|
||||
"type": "find_unpacked_extension_path"
|
||||
},
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules": {
|
||||
"expected": "/home/user/Desktop/helloExtension"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"id": "ce2b64a2-ddc1-4f91-8c7d-a88be7121aac",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "There are several pictures of mountains in my Pictures directory, but I don’t know the names of these mountains. Please help me identify these pictures and change the names of these pictures to the names of the mountains in the pictures.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1LSFS3AI4t5aOIEUby73ZHvURqnx5cw7C",
|
||||
"path": "/home/user/Pictures/picture1.jpg"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1pdRoU_FnqoDgenvp1WzFJZrsGnsexRoo",
|
||||
"path": "/home/user/Pictures/picture2.jpg"
|
||||
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1ajGd9b_YvE1Ay59Pqh8RnePgbnepGc_g",
|
||||
"path": "/home/user/Pictures/picture3.jpg"
|
||||
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome", "os", "image"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig":[
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=1cbBn9BqtkmiZH-nNJAfQ_RYjaAUGHHTG",
|
||||
"path": "/home/user/Desktop/image_script.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "check_direct_json_object",
|
||||
"result":{
|
||||
"type": "vm_command_line",
|
||||
"command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/image_script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py",
|
||||
"shell": "true"
|
||||
},
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected":{
|
||||
"ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": "Kilimanjaro",
|
||||
"999957c8de835bfa420d198270e7a6b079ee20ff53a3f214491e8134768a7c0b": "Himalayas",
|
||||
"79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": "Hua"
|
||||
},
|
||||
"expect_in_result": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"id": "da922383-bfa4-4cd3-bbad-6bebab3d7742",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs you are opening now in pdf format to /home/user/Documents/Blogs.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://lilianweng.github.io/posts/2023-06-23-agent/",
|
||||
"https://lilianweng.github.io/posts/2024-02-05-human-data-quality/"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome", "os"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig":[
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.google.com/uc?export=download&id=19SJ4ydB36lwIK4K4B57sl3bsY9T-3WNv",
|
||||
"path": "/home/user/Desktop/script.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": "pip install PyMuPDF",
|
||||
"shell": "true"
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "exact_match",
|
||||
"result":{
|
||||
"type": "vm_command_line",
|
||||
"command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py",
|
||||
"shell": "true"
|
||||
},
|
||||
"expected": {
|
||||
"type": "rule",
|
||||
"rules":{
|
||||
"expected": "[1, 1]\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"id": "dd60633f-2c72-42ba-8547-6f2c8cb0fdb0",
|
||||
"snapshot": "multiapps",
|
||||
"instruction": "I ran some simple code demos on the currently open google colab, and I think the effect is pretty good. Please help me extract the code in all code boxes, merge it into a \"task.py\" file and store it in the local Home directory.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://colab.research.google.com/github/stanfordnlp/dspy/blob/main/intro.ipynb#scrollTo=L1BHAoL_LRd7"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome", "os", "writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "compare_python_pure_text",
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/colab.py",
|
||||
"dest": "colab.py"
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.google.com/uc?export=download&id=1Z1Vl7SOXf0yjoKttkZD1oYbLM7TgXr5s",
|
||||
"dest": "colab_Gold.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user