lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
tsuky_chen
2024-03-09 18:53:27 +08:00
parent 5b07ec17bf 2291af394f
commit aae848196b
48 changed files with 2106 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
desktop_env/evaluators/getters/__init__.py
View File

@@ -24,6 +24,7 @@ from .chrome import (
get_gotoRecreationPage_and_get_html_content,
get_url_dashPart,
get_active_url_from_accessTree,
get_find_installed_extension_name,
get_info_from_website
)
from .file import get_cloud_file, get_vm_file, get_cache_file, get_content_from_vm_file
@@ -31,7 +32,8 @@ from .general import get_vm_command_line, get_vm_terminal_output, get_vm_command
from .gimp import get_gimp_config_file
from .impress import get_audio_in_slide, get_background_image_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime
from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime, get_time_diff_range
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player
from .vscode import get_vscode_config
from .calc import get_conference_city_in_order

15
desktop_env/evaluators/getters/calc.py Normal file
View File

@@ -0,0 +1,15 @@
import csv
# I want to write a function, reads a csv file, and get all the contents in the third column in the order of rows
def get_conference_city_in_order(env, config):
# read the csv file
csv_path = config['csv_path']
print(f"Reading csv file from {csv_path}")
with open(csv_path, 'r') as f:
reader = csv.reader(f)
# skip the header row
next(reader)
# get the third column in the order of rows
conference_city_list = [row[2] for row in reader]
return conference_city_list

51
desktop_env/evaluators/getters/chrome.py
View File

@@ -4,6 +4,7 @@ import os
import platform
import sqlite3
import time
from urllib.parse import unquote
from typing import Dict, Any, List
from urllib.parse import urlparse, parse_qs
@@ -1010,6 +1011,43 @@ def get_find_unpacked_extension_path(env, config: Dict[str, str]):
return "Google"
def get_find_installed_extension_name(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
if "arm" in platform.machine():
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[
'output'].strip()
else:
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# Preferences store all the path of installed extensions, return them all and let metrics try to find one matches the targeted extension path
all_extensions_name = []
all_extensions = data.get('extensions', {}).get('settings', {})
for id in all_extensions.keys():
name = all_extensions[id]["manifest"]["name"]
all_extensions_name.append(name)
return all_extensions_name
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
def get_data_delete_automacally(env, config: Dict[str, str]):
"""
This function is used to open th "auto-delete" mode of chromium
@@ -1037,8 +1075,8 @@ def get_data_delete_automacally(env, config: Dict[str, str]):
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
data_delete_state = data["profile"]["exit_type"]
return data_delete_state
data_delete_state = data["profile"].get("default_content_setting_values", None)
return "true" if data_delete_state is not None else "false"
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
@@ -1077,6 +1115,7 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]):
"""
active_tab_url = get_active_url_from_accessTree(env, config)
if not isinstance(active_tab_url, str):
logger.error("active_tab_url is not a string")
return None
host = env.vm_ip
port = 9222 # fixme: this port is hard-coded, need to be changed from config file
@@ -1109,12 +1148,14 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]):
for context in browser.contexts:
for page in context.pages:
page.wait_for_load_state("networkidle")
if page.url == active_tab_url:
# the accTree and playwright can get encoding(percent-encoding) characters, we need to convert them to normal characters
if unquote(page.url) == unquote(active_tab_url):
target_page = page
print("tartget page url: ", target_page.url)
print("tartget page title: ", target_page.title())
print("\33[32mtartget page url: ", target_page.url, "\33[0m")
print("\33[32mtartget page title: ", target_page.title(), "\33[0m")
break
if target_page is None:
logger.error("Your tab is not the target tab.")
return {}
return_json = {}
if config["category"] == "class":

25
desktop_env/evaluators/getters/file.py
View File

@@ -1,6 +1,7 @@
import os
from typing import Dict, List, Set
from typing import Optional, Any, Union
from datetime import datetime
import requests
import pandas as pd
@@ -77,21 +78,37 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option
gives (List[int]): optional. defaults to [0]. which files are directly
returned to the metric. if len==1, str is returned; else, list is
returned.
only support for single file now:
time_suffix(bool): optional. defaults to False. if True, append the current time in required format.
time_format(str): optional. defaults to "%Y_%m_%d". format of the time suffix.
"""
time_format = "%Y_%m_%d"
if not config.get("multi", False):
paths: List[str] = [config["path"]]
dests: List[str] = [config["dest"]]
print(config)
if "time_suffix" in config.keys() and config["time_suffix"]:
if "time_format" in config.keys():
time_format = config["time_format"]
# Insert time before . in file type suffix
paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths]
dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests]
else:
paths: List[str] = config["path"]
dests: List[str] = config["dest"]
print(paths)
print(dests)
cache_paths: List[str] = []
gives: Set[int] = set(config.get("gives", [0]))
for i, (p, d) in enumerate(zip(paths, dests)):
print("env cache_dir: ")
print(env.cache_dir)
_path = os.path.join(env.cache_dir, d)
print("_path: ")
print(_path)
file = env.controller.get_file(p)
if file is None:
#return None
@@ -104,7 +121,9 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option
cache_paths.append(_path)
with open(_path, "wb") as f:
f.write(file)
# debug
print("cache_paths")
print(cache_paths)
return cache_paths[0] if len(cache_paths)==1 else cache_paths

7
desktop_env/evaluators/getters/misc.py
View File

@@ -195,3 +195,10 @@ def get_accessibility_tree(env, *args) -> str:
accessibility_tree: str = env.controller.get_accessibility_tree()
logger.debug("AT@eval: %s", accessibility_tree)
return accessibility_tree
def get_time_diff_range(env, config) -> str:
try:
return config["diff_range_in_minutes"]
except:
logger.error("diff_range_in_minutes not found in config.")
return None

22
desktop_env/evaluators/metrics/__init__.py
View File

@@ -2,7 +2,8 @@ from .basic_os import (
check_gnome_favorite_apps,
is_utc_0,
check_text_enlarged,
check_moved_jpgs
check_moved_jpgs,
is_in_vm_clickboard
)
from .chrome import (
is_expected_tabs,
@@ -19,6 +20,7 @@ from .chrome import (
is_expected_active_tab,
is_expected_url_pattern_match,
is_added_to_steam_cart,
is_expected_installed_extensions,
compare_pdf_images
)
from .docs import (
@@ -47,6 +49,7 @@ from .docs import (
check_file_exists,
check_tabstops,
compare_contains_image,
compare_docx_files_and_ignore_new_lines,
compare_docx_images,
compare_image_text,
compare_references
@@ -62,6 +65,14 @@ from .general import (
fuzzy_match,
check_include_exclude,
check_direct_json_object,
compare_time_in_speedtest_results,
is_included_all_json_objects,
is_gold_text_included_in_pdf,
check_csv_line_number,
file_contains,
compare_terminal_and_txt,
fuzzy_place_math,
compare_python_pure_text,
diff_text_file,
literal_match
)
@@ -69,7 +80,7 @@ from .gimp import (
check_brightness_decrease_and_structure_sim,
check_contrast_increase_and_structure_sim,
check_saturation_increase_and_structure_sim,
check_image_size_and_structure_sim,
check_image_size,
check_image_mirror,
check_palette_and_structure_sim,
check_textbox_on_leftside,
@@ -82,7 +93,9 @@ from .gimp import (
increase_saturation,
decrease_brightness,
check_file_exists,
compare_triangle_positions
compare_triangle_positions,
check_sharper,
check_image_file_size
)
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
@@ -126,13 +139,16 @@ from .vscode import (
compare_text_file,
compare_config,
compare_answer,
compare_result_files,
is_extension_installed,
check_json_settings,
check_json_keybindings,
check_python_file_by_test_suite,
check_python_file_by_gold_file,
check_html_background_image,
compare_zip_files
)
from .calc import compare_conference_city_in_order
from .others import compare_epub, check_mp3_meta
def infeasible():

12
desktop_env/evaluators/metrics/basic_os.py
View File

@@ -56,3 +56,15 @@ def check_moved_jpgs(directory_list, rule):
return 1
else:
return 0
def is_in_vm_clickboard(config, terminal_output):
print("terminal_output: ")
print(terminal_output)
print("config: ")
print(config)
expected_results = config["expected"]
# check if terminal_output has expected results
if not isinstance(expected_results, list):
return 1 if expected_results in terminal_output else 0
else:
return 1 if all(result in terminal_output for result in expected_results) else 0

27
desktop_env/evaluators/metrics/calc.py Normal file
View File

@@ -0,0 +1,27 @@
import openpyxl
def compare_conference_city_in_order( actual_city_list_path, expected_city):
expected_city_list = expected_city["expected"]
print(f"Reading csv file from {actual_city_list_path}")
wb = openpyxl.load_workbook(actual_city_list_path)
sheet = wb.active
actual_city_list = []
for row in sheet["C2:C22"]:
for cell in row:
actual_city_list.append(cell.value)
# expected_city is the city that we want to compare with the actual city list
# must in order index
# debug
print("expected_city_list:")
print(expected_city_list)
print("actual_city_list_path:")
print(actual_city_list)
wrong_list = []
try:
for i in range(len(actual_city_list)):
if expected_city_list[i] not in actual_city_list[i]:
wrong_list.append(i)
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
except:
return False
return True if len(wrong_list) == 0 else False

33
desktop_env/evaluators/metrics/chrome.py
View File

@@ -2,9 +2,9 @@ import logging
import os
import re
import shutil
from itertools import product
from typing import Any, Dict, List, Union
import fitz # PyMuPDF
import rapidfuzz.fuzz as fuzz
from bs4 import BeautifulSoup, Tag
@@ -61,6 +61,12 @@ def is_expected_url_pattern_match(result, rules) -> float:
return 1.
def is_expected_installed_extensions(installed_extensions, expected) -> float:
print("installed_extensions: ")
print(installed_extensions)
expected_extensions = expected["expected"]
return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed
def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float:
"""
Checks if the expected tabs are open in Chrome.
@@ -94,12 +100,24 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float:
elif rule['type'] == "liked_authors_websites_urls":
# Check if "liked authors" folder exists
liked_authors_folder = next((bookmark for bookmark in bookmarks['bookmark_bar']['children'] if
bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None)
bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None)
if liked_authors_folder:
# Check if it contains the specified URLs
liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if
bookmark['type'] == 'url']
return 1. if set(liked_authors_urls) == set(rule['urls']) else 0.
urls = rule['urls']
for idx, url in enumerate(urls):
if isinstance(url, str):
urls[idx] = [url]
combinations = product(*urls)
for combination in combinations:
if set(combination) == set(liked_authors_urls):
return 1.
return 0.
else:
return 0.
else:
@@ -140,15 +158,16 @@ def compare_pdfs(pdf1_path: Union[str, List[str]], pdf2_path: Union[str, List[st
logger.info(f"[ERROR]: unexpected error occurred when comparing PDF files: {e}")
return score / len(pdf2_path)
import fitz
from PIL import Image
from io import BytesIO
from borb.pdf import Document
from borb.pdf import PDF
from pathlib import Path
import typing
def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float:
def extract_images_from_pdf(pdf_path):
pdf_document = fitz.open(pdf_path)
@@ -163,14 +182,14 @@ def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float:
images.append(img)
return images
def fix_pdf(in_path: Path, out_path: Path) -> None:
doc: typing.Optional[Document] = None
with open(in_path, "rb") as fh:
doc = PDF.loads(fh)
with open(out_path, "wb") as fh:
PDF.dumps(fh, doc)
fix_pdf(Path(pdf1_path), Path(pdf1_path))
fix_pdf(Path(pdf2_path), Path(pdf2_path))
@@ -183,7 +202,7 @@ def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float:
for img1, img2 in zip(images1, images2):
if img1.tobytes() != img2.tobytes():
return 0.
return 1.

28
desktop_env/evaluators/metrics/demo.py Normal file
View File

@@ -0,0 +1,28 @@
import fitz # PyMuPDF
def extract_answers_from_pdf(pdf_file):
# 打开PDF文件
doc = fitz.open(pdf_file)
answers = []
# 遍历每一页
for page in doc:
# 提取当前页的文本
text = page.get_text()
# 分割文本为行
lines = text.split('\n')
for line in lines:
if line.strip(): # 排除空白行
# 分割等号，提取答案
parts = line.split('=')
if len(parts) > 1:
answer = parts[-1].strip() # 取等号后的部分为答案
answers.append(answer)
return answers
# 假设你的文件名是'math_problems.pdf'
pdf_file = '/Users/lxc/Desktop/calculus.pdf'
answers = extract_answers_from_pdf(pdf_file)
for i, answer in enumerate(answers, 1):
print(f"题目{i}的答案是: {answer}")

176
desktop_env/evaluators/metrics/docs.py
View File

@@ -3,8 +3,10 @@ import os
import re
import xml.etree.ElementTree as ET
import zipfile
from io import BytesIO
from typing import List, Dict, Any
from PIL import Image
from docx import Document
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT, WD_TAB_ALIGNMENT
from docx.shared import RGBColor
@@ -23,6 +25,9 @@ def find_default_font(config_file_path, rules):
default_font = None
expected_font = rules["font_name"]
if not config_file_path:
return 0
try:
tree = ET.parse(config_file_path)
root = tree.getroot()
@@ -42,6 +47,9 @@ def find_default_font(config_file_path, rules):
def contains_page_break(docx_file):
if not docx_file:
return 0
doc = Document(docx_file)
namespaces = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
@@ -62,6 +70,9 @@ def compare_docx_files(file1, file2, **options):
ignore_order = options.get('ignore_order', False)
content_only = options.get('content_only', False)
if not file1 or not file2:
return 0
def get_paragraph_texts_odt(document):
paragraphs = document.getElementsByType(P)
paragraph_texts = []
@@ -118,20 +129,30 @@ def compare_docx_files(file1, file2, **options):
if text1 != text2:
return 0
else:
print("ignore_blanks=false")
if len(doc1_paragraphs) != len(doc2_paragraphs):
print(doc1_paragraphs)
print(doc2_paragraphs)
print(len(doc1_paragraphs))
print(len(doc2_paragraphs))
return 0
print("in compare")
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if ignore_case:
p1, p2 = p1.lower(), p2.lower()
if p1 != p2:
print(p1)
print(p2)
return 0
return 1
def compare_init_lines(file1, file2):
if not file1 or not file2:
return 0
doc1 = Document(file1)
doc2 = Document(file2)
@@ -149,6 +170,9 @@ def compare_init_lines(file1, file2):
def compare_docx_tables(docx_file1, docx_file2):
if not docx_file1 or not docx_file2:
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -174,11 +198,10 @@ def compare_docx_tables(docx_file1, docx_file2):
return 1
from io import BytesIO
from PIL import Image
def compare_docx_images(docx_file1, docx_file2):
if not docx_file1 or not docx_file2:
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -212,6 +235,9 @@ def compare_image_text(image_path, rule):
def compare_line_spacing(docx_file1, docx_file2):
if not docx_file1 or not docx_file2:
return 0
if not compare_docx_files(docx_file1, docx_file2):
return 0
doc1 = Document(docx_file1)
@@ -233,6 +259,9 @@ def compare_line_spacing(docx_file1, docx_file2):
def compare_insert_equation(docx_file1, docx_file2):
if not docx_file1 or not docx_file2:
return 0
if not compare_docx_files(docx_file1, docx_file2):
return 0
@@ -248,6 +277,9 @@ def compare_insert_equation(docx_file1, docx_file2):
def compare_font_names(docx_file, rules: List[Dict[str, Any]]):
if not docx_file:
return 0
doc = Document(docx_file)
expected_font = rules["font_name"]
@@ -260,6 +292,9 @@ def compare_font_names(docx_file, rules: List[Dict[str, Any]]):
def compare_subscript_contains(docx_file1, docx_file2):
if not docx_file1 or not docx_file2:
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -272,6 +307,9 @@ def compare_subscript_contains(docx_file1, docx_file2):
def has_page_numbers_in_footers(docx_file):
if not docx_file:
return 0
doc = Document(docx_file)
for section in doc.sections:
@@ -286,6 +324,9 @@ def has_page_numbers_in_footers(docx_file):
def is_first_line_centered(docx_file):
if not docx_file:
return 0
doc = Document(docx_file)
first_paragraph = doc.paragraphs[0]
@@ -294,11 +335,16 @@ def is_first_line_centered(docx_file):
def check_file_exists(directory, filename):
if not directory or not filename:
return 0
file_path = os.path.join(directory, filename)
return 1 if os.path.isfile(file_path) else 0
def check_tabstops(docx_file1, docx_file2, **kwargs) -> float:
if not docx_file1 or not docx_file2:
return .0
doc1: Document = Document(docx_file1)
doc2: Document = Document(docx_file2)
para1 = [p for p in doc1.paragraphs if p.text.strip()]
@@ -334,6 +380,9 @@ def check_tabstops(docx_file1, docx_file2, **kwargs) -> float:
def compare_contains_image(docx_file1, docx_file2):
if not docx_file1 or not docx_file2:
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -346,6 +395,9 @@ def compare_contains_image(docx_file1, docx_file2):
def evaluate_colored_words_in_tables(file_path1, file_path2, **kwargs):
if not file_path1 or not file_path2:
return 0
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
@@ -380,6 +432,9 @@ def evaluate_colored_words_in_tables(file_path1, file_path2, **kwargs):
def check_highlighted_words(file_path1, file_path2):
if not file_path1 or not file_path2:
return 0
if not compare_docx_files(file_path1, file_path2):
return 0
@@ -402,6 +457,9 @@ def check_highlighted_words(file_path1, file_path2):
def evaluate_strike_through_last_paragraph(file_path1, file_path2):
if not file_path1 or not file_path2:
return 0
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
@@ -418,6 +476,9 @@ def evaluate_strike_through_last_paragraph(file_path1, file_path2):
def evaluate_conversion(file_path):
if not file_path:
return 0
document = Document(file_path)
for table in document.tables:
@@ -437,6 +498,9 @@ def evaluate_conversion(file_path):
def evaluate_spacing(file_path):
if not file_path:
return 0
document = Document(file_path)
# Check line spacing for introduction, body, and conclusion
@@ -450,6 +514,9 @@ def evaluate_spacing(file_path):
def check_italic_font_size_14(path1, path2):
if not path1 or not path2:
return 0
if not compare_docx_files(path1, path2):
return 0
document = Document(path1)
@@ -463,6 +530,9 @@ def check_italic_font_size_14(path1, path2):
def evaluate_alignment(docx_path):
if not docx_path:
return 0
# Load the document
doc = Document(docx_path)
@@ -492,6 +562,9 @@ def evaluate_alignment(docx_path):
def get_unique_train_ids(initial_file): # fixed standard
if not initial_file:
return set(), 0
doc = Document(initial_file)
train_ids = set()
processed_lines = 0
@@ -508,6 +581,9 @@ def get_unique_train_ids(initial_file): # fixed standard
def check_no_duplicates(initial_file, processed_file):
if not initial_file or not processed_file:
return 0
# Open the document
train_ids_ini, ini_lines = get_unique_train_ids(initial_file)
doc_processed = Document(processed_file)
@@ -535,6 +611,9 @@ def check_no_duplicates(initial_file, processed_file):
def compare_docx_lines(file1, file2):
if not file1 or not file2:
return 0
# Read the text of the document, line by line
doc1 = Document(file1)
doc1_lines = [p.text.strip() for p in doc1.paragraphs if p.text.strip()]
@@ -551,8 +630,47 @@ def compare_docx_lines(file1, file2):
return 0
def compare_docx_files_and_ignore_new_lines(file1, file2, **options):
ignore_blanks = options.get('ignore_blanks', True)
if not file1 or not file2:
return 0
# Determine file types and load documents
if file1.endswith('.docx') and file2.endswith('.docx'):
doc1 = Document(file1)
doc2 = Document(file2)
# First, delete all the blank in paragraphs
doc1 = [p for p in doc1.paragraphs if p.text != '']
doc2 = [p for p in doc2.paragraphs if p.text != '']
doc1_paragraphs = [p.text for p in doc1]
doc2_paragraphs = [p.text for p in doc2]
else:
# Unsupported file types or mismatch
print("Unsupported file types or mismatch between file types.")
return 0
# Process and compare documents
if ignore_blanks:
text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
if text1 != text2:
return 0
else:
if len(doc1_paragraphs) != len(doc2_paragraphs):
return 0
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if p1 != p2:
return 0
return 1
# Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated
def compare_highlighted_text(file1, file2):
if not file1 or not file2:
return 0
def extract_highlighted_text(file_path):
highlighted_texts = []
@@ -590,6 +708,9 @@ def compare_highlighted_text(file1, file2):
def compare_references(file1, file2, **options):
if not file1 or not file2:
return 0
reference_indicator = options.get('reference_indicator', 'References')
reference_base_result = options.get('reference_base_result', 0.5)
@@ -634,48 +755,3 @@ def compare_references(file1, file2, **options):
return (result - reference_base_result) / (1 - reference_base_result)
else:
return 0
def compare_answer(file1, file2, **options):
"""This is a specific function to compare the """
# Determine file types and load documents
if file1.endswith('.docx') and file2.endswith('.docx'):
doc1 = Document(file1)
doc2 = Document(file2)
doc1_paragraphs = [p.text for p in doc1.paragraphs]
doc2_paragraphs = [p.text for p in doc2.paragraphs]
else:
# Unsupported file types or mismatch
print("Unsupported file types or mismatch between file types.")
return 0
# Find the references section in the paragraphs, find the idx of the last reference_indicator in the paragraph list
ref1_idx = doc1_paragraphs.index(reference_indicator) if reference_indicator in doc1_paragraphs else -1
ref2_idx = doc2_paragraphs.index(reference_indicator) if reference_indicator in doc2_paragraphs else -1
if ref1_idx == -1 and ref2_idx == -1:
return 1
if ref1_idx == -1 or ref2_idx == -1:
return 0
# split the reference section into reference items, and remove the empty string items
ref1 = [p for p in doc1_paragraphs[ref1_idx + 1:] if p.strip()]
ref2 = [p for p in doc2_paragraphs[ref2_idx + 1:] if p.strip()]
# Compare the references
if len(ref1) != len(ref2):
return 0
total_similarity = 0
for r1, r2 in zip(ref1, ref2):
# fuzzy match the references
similarity = fuzz.ratio(r1, r2) / 100.0
total_similarity += similarity
result = total_similarity / len(ref1)
if result >= reference_base_result:
return (result - reference_base_result) / (1 - reference_base_result)
else:
return 0

192
desktop_env/evaluators/metrics/general.py
View File

@@ -1,7 +1,9 @@
import csv
import datetime
import difflib
import functools
import json
import yaml
import logging
import operator
import re
import sqlite3
@@ -10,17 +12,19 @@ from typing import Callable, Any, Union
from typing import Dict, List, Pattern
import lxml.etree
import pandas as pd
import pdfplumber
import yaml
from docx import Document
from lxml.cssselect import CSSSelector
from lxml.etree import _Element
from rapidfuzz import fuzz
import difflib
from .utils import _match_record, _match_value_to_rule
import logging
logger = logging.getLogger("desktopenv.metric.general")
def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
if result is None:
return 0.
@@ -68,6 +72,7 @@ def is_in_list(result, rules) -> float:
else:
return 0.
def diff_text_file(result: str, expect: str) -> float:
if result is None:
return 0.
@@ -78,12 +83,32 @@ def diff_text_file(result: str, expect: str) -> float:
expected_lines: List[str] = f.read().splitlines()
return difflib.SequenceMatcher(a=result_lines, b=expected_lines).ratio()
def fuzzy_match(result, rules) -> float:
expect = rules["expected"]
return fuzz.ratio(result, expect) / 100.
def fuzzy_place_math(result_file_path, rules) -> float:
expect = rules["expected"] # a list of possible answers
# read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line
doc = Document(result_file_path)
words_list = []
for para in doc.paragraphs:
words_list.extend(para.text.split())
# Print out the list of extracted words
print("Your Answers: ")
print(words_list)
fuzzy_score_list = []
for word in words_list:
max_score = 0
for ans in expect:
score = fuzz.ratio(word, ans)
max_score = max(max_score, score)
fuzzy_score_list.append(max_score)
return sum(fuzzy_score_list) / len(fuzzy_score_list)
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
Args:
@@ -191,10 +216,10 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
return 0.
if "text" in r:
match_func: Callable[[str], Number] = functools.partial( operator.eq if r["exact"] \
else (lambda a, b: fuzz.ratio(a, b) / 100.)
, r["text"]
)
match_func: Callable[[str], Number] = functools.partial(operator.eq if r["exact"] \
else (lambda a, b: fuzz.ratio(a, b) / 100.)
, r["text"]
)
match_score: Number = 0
for elm in elements:
match_score = max(match_score, match_func(elm.text or None))
@@ -267,20 +292,157 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
return float(metric)
def check_direct_json_object(result, rules)->float:
def check_direct_json_object(result, rules) -> float:
"""
One of the most commonly used function to evalute.
Compare two json objects directly.
"""
if isinstance(result, str):
# remove blanks before and after result
result = result.strip()
# replace all ' with "
result = result.replace("'", '"')
# load json object
result = json.loads(result)
print("result: ")
print(result)
print("expected: ")
print(rules["expected"])
if result is None:
return 0.
expected_json = rules["expected"]
for key in expected_json.keys():
expected_value = expected_json.get(key)
if expected_value != result.get(key):
return 0.
return 1.0
expect_in_result = rules.get("expect_in_result", False)
if not expect_in_result:
expected_json = rules["expected"]
for key in expected_json.keys():
expected_value = expected_json.get(key)
if expected_value != result.get(key):
return 0.
return 1.0
else:
expected_json = rules["expected"]
for key in expected_json.keys():
expected_value = expected_json.get(key)
if expected_value not in result.get(key):
return 0.
return 1.0
def compare_time_in_speedtest_results(speedtest_result_path, time_diff):
if not speedtest_result_path:
return 0
# open the speedtest results file(csv)
date_col = None
with open(speedtest_result_path, 'r') as f:
reader = pd.read_csv(f)
for column in reader.columns:
if column.startswith('TEST_DATE'):
date_col = column
break
now_date_time = datetime.datetime.now().strftime('%H:%M')
for date in reader[date_col]:
date_time = date[-5:]
# compare the date time with the current date time, if time diff less than time_diff para, then return true
if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time,
'%H:%M')).total_seconds()) / 60 < int(
time_diff):
return 0
return 1
def is_included_all_json_objects(gold_file_path, result_file_path):
if not gold_file_path or not result_file_path:
return 0
print("gold_file_path: ")
print(gold_file_path)
print("result_file_path: ")
print(result_file_path)
# two json file, check if all the key-value pair in gold_file_path is included in result_file_path
with open(gold_file_path, 'r') as f:
gold_json = json.load(f)
with open(result_file_path, 'r') as fr:
result_json = json.load(fr)
for key in gold_json.keys():
if key not in result_json.keys() or gold_json[key] != result_json[key]:
return 0
return 1
def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path):
print("gold_text_path: ")
print(gold_text_path)
print("pdf_file_path: ")
print(pdf_file_path)
# gold file is a json file, we need to check all the value in json are included in pdf file.
with open(gold_text_path, 'r') as f:
gold_json = json.load(f)
with pdfplumber.open(pdf_file_path) as pdf:
text = ''
for page in pdf.pages:
text += page.extract_text()
false_list = []
for key in gold_json.keys():
if gold_json[key] not in text:
false_list.append(key)
if len(false_list) > 0:
print("false_list: ")
print(false_list)
return 0
else:
return 1
def file_contains(file_path, config):
# file_path ends with .txt
if not file_path:
return 1
with open(file_path, 'r') as f:
file_text = f.read()
for text in config["expected"]:
if text not in file_text:
return 0
return 1
def check_csv_line_number(file_path, line_number):
# check file_path suffix
if not file_path.endswith('.csv'):
return 0
# check line number
with open(file_path, 'r') as f:
reader = csv.reader(f)
line_count = sum(1 for row in reader)
return 1 if line_count == int(line_number["expected"]) else 0
def compare_terminal_and_txt(txt_file_path, terminal_output):
# read txt file content
with open(txt_file_path, 'r') as f:
txt_file_content = f.read()
# compare terminal output with txt file content
return 1 if terminal_output == txt_file_content else 0
def compare_python_pure_text(py_file_path, gold_file_path):
# first, change the suffix of gold_file from .txt to .py
print("py_file_path: ")
print(py_file_path)
print("gold_file_path: ")
print(gold_file_path)
# gold_file_path = gold_file_path.replace('.txt', '.py')
def remove_whitespace(text):
return ''.join(text.split())
with open(py_file_path, 'r') as file1:
content1 = file1.read()
with open(gold_file_path, 'r') as file2:
content2 = file2.read()
content1_no_whitespace = remove_whitespace(content1)
content2_no_whitespace = remove_whitespace(content2)
if content1_no_whitespace == content2_no_whitespace:
return 1
else:
return 0

102
desktop_env/evaluators/metrics/gimp.py
View File

@@ -5,7 +5,7 @@ from PIL import Image, ImageChops, ImageStat
def compare_image_list(pred_img_path_list: Union[str, List[str]],
gold_img_path_list: Union[str, List[str]]) -> float:
gold_img_path_list: Union[str, List[str]]) -> float:
""" Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0
"""
if type(pred_img_path_list) != list:
@@ -177,6 +177,16 @@ def calculate_contrast(image):
return np.std(pixels)
def calculate_image_sharpness(image_path):
# Load the image in grayscale
image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
# Apply the Laplacian operator
laplacian = cv2.Laplacian(image, cv2.CV_64F)
# Calculate the variance
variance = np.var(laplacian)
return variance
def structure_check_by_mse(img1, img2, threshold=0.03):
"""Check if two images are approximately the same by MSE"""
mse = np.mean(
@@ -295,7 +305,8 @@ def check_triangle_position(tgt_path):
# We assume the triangle is a different color from the background
# Find the unique colors
unique_colors, counts = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0, return_counts=True)
unique_colors, counts = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0,
return_counts=True)
unique_colors_sorted = unique_colors[np.argsort(counts)]
# Assuming the background is the most common color and the triangle is a different color
@@ -337,6 +348,25 @@ def check_structure_sim(src_path, tgt_path):
return structure_same
def check_structure_sim_resized(src_path, tgt_path):
"""
Check if the structure of the two images are similar after resizing.
gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15
"""
if src_path is None or tgt_path is None:
return 0.
img_src = Image.open(src_path)
img_tgt = Image.open(tgt_path)
# Resize the images to the same size
img_src = img_src.resize(img_tgt.size)
# Check if the structure is similar
structure_same = structure_check_by_ssim(img_src, img_tgt)
return structure_same
def check_contrast_increase_and_structure_sim(src_path, tgt_path):
"""
Check if the src image has higher contrast than the tgt image and the structures are similar
@@ -388,34 +418,28 @@ def check_config_status(actual_config_path, rule):
return 0.
def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None):
def check_image_size(src_path, rule):
"""
Check if the size of the src image is correct and the structure of the two images are similar.
gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15
Check if the size of the src image is correct
multi-apps:42f4d1c7-4521-4161-b646-0a8934e36081
"""
if src_path is None or tgt_path is None:
if src_path is None:
return 0.
# Load images
source_image = Image.open(src_path)
target_image = Image.open(tgt_path)
# Load the image
img = Image.open(src_path)
# Check size
if width is not None:
width_same = source_image.size[0] == width
else:
width_same = True
if height is not None:
height_same = source_image.size[1] == height
# Check the size
if rule["height"] is not None:
height_same = img.size[1] == rule["height"]
else:
height_same = True
if rule["width"] is not None:
width_same = img.size[0] == rule["width"]
else:
width_same = True
# Check structure
resized_target_image = target_image.resize(source_image.size)
structure_same = structure_check_by_ssim(source_image, resized_target_image)
if width_same and height_same and structure_same:
if height_same and width_same:
return 1.
else:
return 0.
@@ -521,6 +545,31 @@ def check_green_background(src_path, tgt_path):
return 1.
def check_sharper(src_path, tgt_path):
"""
Check if the source image is sharper than the target image.
multi-app:bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108
"""
sharpness_src = calculate_image_sharpness(src_path)
sharpness_tgt = calculate_image_sharpness(tgt_path)
return 1.0 if sharpness_src > sharpness_tgt else 0.0
def check_image_file_size(src_path, rule):
"""
Check if the size of the src image within 500KB
"""
if src_path is None:
return 0.0
# Check the size
file_size = os.path.getsize(src_path)
if file_size < rule["max_size"]:
return 1.0
else:
return 0.0
if __name__ == "__main__":
actual_config_path = "../../../cache/sessionrc_test"
rule = {
@@ -550,3 +599,12 @@ if __name__ == "__main__":
tgt_path = "../../../cache/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce/Triangle_In_The_Middle.png"
print(check_triangle_position(tgt_path))
src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi_sharper.png"
tgt_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi.png"
print(check_sharper(src_path, tgt_path))
src_path = "../../../cache/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f/compressed.jpeg"
rule = {
"max_size": 500000
}
print(check_image_file_size(src_path, rule))

2
desktop_env/evaluators/metrics/utils.py
View File

@@ -1,10 +1,12 @@
import builtins
import datetime
import functools
import itertools
import logging
import operator
import re
import zipfile
import pandas as pd
from typing import Any, TypeVar, Union, Iterable, Optional, Callable
from typing import Dict, List, Set, Match, Tuple, Pattern
from urllib.parse import urlparse, urlunparse

51
desktop_env/evaluators/metrics/vscode.py
View File

@@ -229,3 +229,54 @@ def check_python_file_by_test_suite(actual_files, test_file, **options) -> float
def check_python_file_by_gold_file(actual_files, gold_file: str, **options) -> float:
pass
def check_html_background_image(src_path: str, rule: Dict = None) -> float:
"""
Check if the background image is correctly set.
multi-app:bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108
"""
from bs4 import BeautifulSoup
with open(src_path, 'r') as f:
html_content = f.read()
soup = BeautifulSoup(html_content, 'html.parser')
styles = soup.find_all('style')
for style in styles:
if f'background-image: url(\'{rule["value"]}\')' in style.text:
return 1.0
return 0.0
def compare_result_files(src_path, tgt_path):
"""
Compare whether the content of two files are the same.
multi-app:7f35355e-02a6-45b5-b140-f0be698bcf85
"""
with open(src_path, 'r') as f:
src_content = f.read().strip()
with open(tgt_path, 'r') as f:
tgt_content = f.read().strip()
try:
# Compare the content as numbers
tgt_content_num = float(tgt_content)
if tgt_content in src_content:
# If the content of tgt is in src, return 1.0 since output src might be
# a superset(language description+number) of tgt
return 1.0
src_content_num = float(src_content)
if abs(src_content_num - tgt_content_num) < 1e-4:
return 1.0
return 0.0
except:
if src_content == tgt_content:
return 1.0
return 0.0
if __name__ == "__main__":
src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/index.html"
rule = {
"type:": "value",
"value": "anmi_sharper.png"
}
print(check_html_background_image(src_path, rule))

25
evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json
View File

@@ -53,17 +53,32 @@
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"func": ["is_expected_active_tab", "is_expected_active_tab"],
"conj": "or",
"result": [
{
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
},
{
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
}
],
"expected": [
{
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.drugs.com/npc/"
}
}
},
{
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.drugs.com/npp/"
}
}]
}
}

4
evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json
View File

@@ -62,8 +62,8 @@
"type": "rule",
"rules":{
"expected": {
"locationName": "Zurich Airport",
"dropLocationName": "Zurich Airport",
"locationName": "Zürich",
"dropLocationName": "Zürich",
"filterCriteria_carCategory": "large",
"filterCriteria_sortBy": "PRICE"
}

2
evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json
View File

@@ -1,7 +1,7 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "chrome",
"instruction": "Can you make a new folder for me on that bookmarks bar in my internet browser? Let's call it 'Favorites.'",
"instruction": "Can you make a new folder for me on the bookmarks bar in my internet browser? Let's call it 'Favorites.'",
"source": "https://www.youtube.com/watch?v=IN-Eq_UripQ",
"config": [
{

2
evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json
View File

@@ -54,7 +54,7 @@
"rules":{
"expected": {
"q": "drip coffee maker",
"tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,pdtr0:1825161|1825162"
"tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,sales:1,pdtr0:1825161|1825162"
}
}
}

11
evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json
View File

@@ -1,7 +1,7 @@
{
"id": "99146c54-4f37-4ab8-9327-5f3291665e1e",
"snapshot": "chrome",
"instruction": "Please help me set Chrome to delete my browsing history automatically every time I close the browser.",
"instruction": "Please help me set Chrome to delete my browsing data automatically every time I close the browser.",
"source": "https://www.youtube.com/watch?v=v0kxqB7Xa6I",
"config": [
{
@@ -29,6 +29,13 @@
"chrome"
],
"evaluator": {
"postconfig":[{
"type": "execute",
"parameters": {
"command": "pkill chrome",
"shell": "true"
}
}],
"func": "exact_match",
"result": {
"type": "data_delete_automacally"
@@ -36,7 +43,7 @@
"expected": {
"type": "rule",
"rules": {
"expected": "Crashed"
"expected": "true"
}
}
}

26
evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json
View File

@@ -43,19 +43,35 @@
"chrome"
],
"evaluator": {
"func": "exact_match",
"result": {
"func": ["exact_match", "exact_match"],
"conj": "or",
"result": [
{
"type": "url_dashPart",
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": false,
"returnType": "string"
},
"expected": {
},
{
"type": "url_dashPart",
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": false,
"returnType": "string"
}],
"expected": [
{
"type": "rule",
"rules": {
"expected": "tamiflu.html#side-effects"
}
}
},
{
"type": "rule",
"rules": {
"expected": "tamiflu-side-effects.html"
}
}]
}
}

2
evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json
View File

@@ -53,7 +53,7 @@
"rules": {
"expected": [
"AgeAppropriate:Kids",
"search=spider-man%20toys",
"search=spider[-%20]?man%20toys",
"S=4"
]
}

32
evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json
View File

@@ -84,16 +84,34 @@
}
}
],
"func": "check_image_size_and_structure_sim",
"expected":{
"func": [
"check_image_size",
"check_structure_sim"
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/dog_with_background.png",
"dest": "dog_with_background.png"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
{
"type": "vm_file",
"path": "/home/user/Desktop/dog_with_background.png",
"dest": "dog_with_background.png"
}
],
"result": [
{
"type": "rule",
"rules": {
"height": 512
}
},
{
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
]
}
}

49
evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json Normal file
View File

@@ -0,0 +1,49 @@
{
"id": "2373b66a-092d-44cb-bfd7-82e86e7a3b4d",
"snapshot": "multiapps",
"instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor the system CPU usage, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop, and convert the txt to csv file with the same name.",
"source": "author",
"config": [
{
"type": "command",
"parameters":{
"command": "echo password | sudo -S apt-get update && echo password | sudo -S apt-get install sysstat",
"shell": "true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "calc"
],
"evaluator": {
"func": ["file_contains", "check_csv_line_number"],
"result":
[
{
"type": "vm_file",
"path": "/home/user/Desktop/System_Resources_Report.txt",
"dest": "System_Resources_Report.txt"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/System_Resources_Report.csv",
"dest": "System_Resources_Report.csv"
}
],
"expected":
[
{
"type": "rule",
"rules" :{
"expected": ["tps", "rtps", "wtps", "dtps", "bread/s", "bwrtn/s", "bdscd/s", "Average", "Linux"]
}
},
{
"type": "rule",
"rules": {
"expected": "33"
}
}]
}
}

86
evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json
View File

@@ -1,25 +1,69 @@
{
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"snapshot": "libreoffice_calc",
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed.",
"source": "authors",
"config": [
],
"trajectory": "trajectories/26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"related_apps": [
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"snapshot": "multiapps",
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed (if the dir does not exist, create it).",
"source": "https://www.speedtest.net/",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
"expected": {
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
"options": {
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.speedtest.net/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps":[
"os",
"browser"
],
"evaluator":{
"func": "compare_time_in_speedtest_results",
"result":{
"type": "vm_file",
"path": "/home/user/Test/Speed/Speedtest Results Export-.csv",
"dest": "Speedtest Results Export-.csv",
"time_suffix": true
},
"expected":{
"type": "time_diff_range",
"diff_range_in_minutes": "60"
}
}
}
}
}

37
evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json Normal file
View File

@@ -0,0 +1,37 @@
{
"id": "3c8f201a-009d-4bbe-8b65-a6f8b35bb57f",
"snapshot": "gimp",
"instruction": "Use `gdown` to download the image from \"https://drive.google.com/uc?export=download&id=1i8j5dGS57sA07jEuPNAlQW-sn5uqUnuK\", and then use GIMP to compress it to under 600KB. Resize if needed.",
"source": "",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_image_file_size",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/compressed.jpeg",
"dest": "compressed.jpeg"
},
"expected": {
"type": "rule",
"rules": {
"max_size": 600000
}
}
}
}

52
evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json Normal file
View File

@@ -0,0 +1,52 @@
{
"id": "3eb2a122-a5e3-4f89-9820-f7fa1a582969",
"snapshot": "multiapps",
"instruction": "Please search online for the submission deadline and venue of the ICLR main conference in 2035, and copy it to my clipboard. If not yet publicized, copy None.",
"source": "author",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "command",
"parameters":{
"command": "echo password | sudo -S apt install xsel && xsel -bc",
"shell": "true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "chrome"
],
"evaluator": {
"func": "is_in_vm_clickboard",
"expected": {
"type": "vm_command_line",
"command": "xsel --clipboard --output",
"shell": "true"
},
"result": {
"type": "rule",
"rules": {
"expected": ["None"]
}
}
}
}

71
evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "42f4d1c7-4521-4161-b646-0a8934e36081",
"snapshot": "gimp",
"instruction": "Configure VS Code to edit GIMP script-fu scripts effectively by installing lisp extension. Test by writing code to resizing the image as 128 * 128 as \"resized.png\"",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1yrWU5HimYPNUjdtvw1a218kh50fPVtZ3",
"path": "/home/user/Desktop/character.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"is_extension_installed",
"check_image_size"
],
"result": [
{
"type": "vm_command_line",
"command": [
"code",
"--list-extensions",
"|",
"grep",
"mattn.lisp"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
],
"expected": [
{
"type": "rule",
"rules": {
"type": "contain",
"expected": "mattn.lisp"
}
},
{
"type": "rule",
"rules": {
"height": 128,
"width": 128
}
}
]
}
}

70
evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json Normal file
View File

@@ -0,0 +1,70 @@
{
"id": "5bc63fb9-276a-4439-a7c1-9dc76401737f",
"snapshot": "multiapps",
"instruction": "I have a JSON-formatted data file opened now that stores the responses of several large language models on a specific dataset. Now, I want to filter out all the responses from Gemini and specifically look at the sentences in the responses that contain \"Iliad\". Please copy all of Gemini's responses(values) from the JSON file, paste them into a Word document named \"gemini_results.docx\" under Desktop. Each response should be a new paragraph and sepreated by a new line. Highlight the all the \"Iliad\" word.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1OtN8poSktw5s9w7y28wr5JmXszMOpn2q",
"path": "/home/user/Desktop/llm_answers.json"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/llm_answers.json"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "gemini_results.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_docx_files_and_ignore_new_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1gmg5mC7qfmPvspC-ICOIJJ3aRjuSvP8u",
"dest": "gemini_results_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/gemini_results.docx",
"dest": "gemini_results.docx"
},
"options": {
"ignore_blanks": false
}
}
}

70
evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json Normal file
View File

@@ -0,0 +1,70 @@
{
"id": "65e62ec6-4603-4c67-b4d2-07830deb285b",
"snapshot": "multiapps",
"instruction": "I've recently become very interested in large language models. I tried entering some questions into several large language models and stored the answers in a file named \"answers.json\" under Desktop. Now, I want to focus only on the answers from GPT-4. Please help me copy all the GPT-4 answers into a Word document named \"results.docx\" under Desktop, one answer per paragraph.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6",
"path": "/home/user/Desktop/answers.json"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/answers.json"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "results.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_docx_files_and_ignore_new_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1edtU4kHecg7oxerf1jvd1-ak-2WmEla3",
"dest": "results_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/results.docx",
"dest": "results.docx"
},
"options": {
"ignore_blanks": false
}
}
}

72
evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json Normal file
View File

@@ -0,0 +1,72 @@
{
"id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a",
"snapshot": "multiapps",
"instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.",
"source": "author",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH",
"path": "/home/user/Desktop/ConferenceCity.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/ConferenceCity.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"calc", "chrome", "os"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP",
"path": "/home/user/Desktop/ConferenceCity_Gold.xlsx"
}
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "ConferenceCity.xlsx - LibreOffice Calc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": "compare_conference_city_in_order",
"expected": {
"type": "rule",
"rules":{
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"]
}
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/ConferenceCity.xlsx",
"dest": "ConferenceCity.xlsx"
}
}
}

51
evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "716a6079-22da-47f1-ba73-c9d58f986a38",
"snapshot": "multiapps",
"instruction": "I remember there is a file named \"secret.docx\" on this computer, but I can't remember where it is. Please find the path where this file is stored and copy it to the clipboard.",
"source": "",
"config": [
{
"type": "command",
"parameters": {
"command": "mkdir -p /home/user/Data1/List1 && mkdir -p /home/user/Data2/List2 && mkdir -p /home/user/Data3/List3 && mkdir -p /home/user/Data4/List4 && mkdir -p /home/user/Data5/List5",
"shell" :"true"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1iaqtXHZs2sA11nmoLHPK9VXGO0_6xq2l",
"path": "/home/user/Data3/List3/secret.docx"
}
]
}
},
{
"type": "command",
"parameters":{
"command": "echo password | sudo -S apt install xsel && xsel -bc",
"shell": "true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "terminal"
],
"evaluator": {
"func": "is_in_vm_clickboard",
"expected": {
"type": "vm_command_line",
"command": "xsel --clipboard --output",
"shell": "true"
},
"result": {
"type": "rule",
"rules": {
"expected": "/home/user/Data3/List3/secret.docx"
}
}
}
}

43
evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json Normal file
View File

@@ -0,0 +1,43 @@
{
"id": "7f35355e-02a6-45b5-b140-f0be698bcf85",
"snapshot": "libreoffice_calc",
"instruction": "Export the table to a CSV file and then help me write code to find the medium price (fill empty value with average). Save the result in \"result.txt\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://docs.google.com/spreadsheets/d/13YL-KC__pav2qp3sFDs1BT2wZnpWGp7s/export?format=xlsx",
"path": "/home/user/Desktop/stock.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/stock.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"vs_code"
],
"evaluator": {
"func": "compare_result_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/result.txt",
"dest": "result.txt"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1oPPW_dozWGII5MRmdXdKKoEK5iBkd_8Q",
"dest": "result_gold.txt"
}
}
}

83
evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json Normal file
View File

@@ -0,0 +1,83 @@
{
"id": "7ff48d5b-2df2-49da-b500-a5150ffc7f18",
"snapshot": "multiapps",
"instruction": "I am a Chinese citizen and I want to go to Macau to watch a concert recently, but I have not yet applied for a visa for Macau. I live in Futian District, Shenzhen City. I heard that Shenzhen currently has 24-hour self-service check-in machines. Please help me find the addresses of 5 24-hour self-service check-in machines in Futian District and save them in Chinese in this open word document.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1H-GVXzyn30zxBKw02w5p5ZQzcwEZSlE4&export=download&authuser=0&confirm=t&uuid=da987f7a-7aa5-43d9-8426-741a97393726&at=APZUnTWyce0qNBfoOjOZ5pNRwIvX:1709971794981",
"path": "/home/user/Desktop/AllLocations.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/AllLocations.docx"
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "AllLocations.docx - LibreOffice Writer"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os", "writer"
],
"evaluator": {
"func": "fuzzy_place_math",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/AllLocations.docx",
"dest": "AllLocations.docx"
},
"expected": {
"type": "rule",
"rules":{
"expected": [
"深圳市福田区益田路5055号信息枢纽大厦西门一楼",
"深圳市福田区福华三路111号北三门会展中心警务室",
"深圳市福田区正义街1号",
"福田区莲科路18号莲花一村警务室",
"深圳市福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边",
"深圳市福田区香梅路2002-4号",
"福田区水围村龙景楼一楼",
"深圳市福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面",
"深圳市福田区福强路3028号金沙嘴大厦",
"深圳市福田区天安数码城昌泰公寓一楼",
"福田区泰然五路5号天安数码城9栋",
"深圳市福田区振兴路108号",
"深圳市福田区滨河大道2033号",
"深圳市福田区上沙四十八栋一巷11",
"深圳市福田区北环大道与香蜜湖路交汇处香蜜原著警务室",
"深圳市福田区八卦路38号八卦岭派出所",
"深圳市福田区宝能城市公馆B栋一楼竹园警务室",
"深圳市福田区竹子林五路12号",
"福田区福强路3028号金沙嘴大厦",
"福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边",
"福田区益田路5055号信息枢纽大厦西门一楼",
"福田区正义街1号",
"福田区香梅路2002-4号",
"福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面",
"福田区天安数码城昌泰公寓一楼",
"福田区振兴路108号",
"福田区滨河大道2033号",
"福田区上沙四十八栋一巷11",
"福田区北环大道与香蜜湖路交汇处香蜜原著警务室",
"福田区八卦路38号八卦岭派出所",
"福田区宝能城市公馆B栋一楼竹园警务室",
"福田区竹子林五路12号"
]
}
}
}
}

66
evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json Normal file
View File

@@ -0,0 +1,66 @@
{
"id": "873cafdd-a581-47f6-8b33-b9696ddb7b05",
"snapshot": "multiapps",
"instruction": "My friend is a \"plugin guru\" and he recommended some good plug-ins to me. Please go to the Chrome plug-in store and install all the listed plug-ins.",
"source": "author",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1EMpoFNW3reLgZE0lj6scRFmL-OPjWuzf",
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Recommended_plugin_list.docx - LibreOffice Writer"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "chrome", "Writer"
],
"evaluator": {
"func": "is_expected_installed_extensions",
"expected": {
"type": "rule",
"rules":{
"expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"]
}
},
"result": {
"type": "find_installed_extension_name"
}
}
}

51
evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "91190194-f406-4cd6-b3f9-c43fac942b22",
"snapshot": "gimp",
"instruction": "Launch GIMP from the command line to edit \"cola.png\" and crop the top 20% off the image for my avatar as \"cropped.png\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1bmSRNNh4JkF6izrKrmynUHarf0pFES50",
"path": "/home/user/Desktop/cola.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1MayrIPJWRK7cMEVe3TxYmgkAbVMrYcQA",
"path": "/home/user/Desktop/cropped_gold.png"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_structure_sim",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/cropped.png",
"dest": "cropped.png"
},
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/cropped_gold.png",
"dest": "cropped_gold.png"
}
}
}

54
evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json Normal file
View File

@@ -0,0 +1,54 @@
{
"id": "98e8e339-5f91-4ed2-b2b2-12647cb134f4",
"snapshot": "vs_code",
"instruction": "Merge the contents of all .txt files from your vscode project into a single document in Writer. No merging separator is needed. Ensure to set the overall font size of the document to 10.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1dqbi6j3zPOqFTMFHVVaEOXSdHz5zIey1",
"path": "/home/user/Desktop/doc_proc.zip"
}
]
}
},
{
"type": "command",
"parameters": {
"command": "mkdir -p /home/user/Desktop/doc_proc/; unzip /home/user/Desktop/doc_proc.zip -d /home/user/Desktop/doc_proc/",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"code",
"-g",
"/home/user/Desktop/doc_proc/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vs_code",
"libreoffice_writer"
],
"evaluator": {
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/concat.docx",
"dest": "concat.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=10k_8T7Hk9KQ2I-AbNK56hvXR6yv0Pemc",
"dest": "concat_gold.docx"
}
}
}

81
evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json Normal file
View File

@@ -0,0 +1,81 @@
{
"id": "9f3bb592-209d-43bc-bb47-d77d9df56504",
"snapshot": "multiapps",
"instruction": "I downloaded a video to practice listening, but I don't know how to remove the subtitles. Please help me remove the subtitles from the video and export it as \"subtitles.srt\" and store it in the same directory as the video.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=17vIS65RVA7e9n5u8lLNqwjBl56MQpkk8&export=download&authuser=0&confirm=t&uuid=1c8882db-dd55-450d-a998-52c03bfaf801&at=APZUnTWPObUX-3Pqtusro4T5Ga5B:1709972109572",
"path": "/home/user/video.mp4"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/video.mp4"
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "video.mp4 - VLC media player"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "vlc"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1oMCDx2hqj0xzkQP4le0Mn09y-vvCYXlt&export=download&authuser=0&confirm=t&uuid=2a2cac5c-f031-41f7-8504-4a013c02ee15&at=APZUnTUa188AtQ93qZG8HInS_o90:1709972317568",
"path": "/home/user/subtitles_Gold.srt"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=19ooP4gPY2HYQVI9n31CtMlJ3q-1dr6cU&export=download&authuser=0&confirm=t&uuid=ba968219-3a9b-44d1-bee6-64e289ce4ffe&at=APZUnTXlL0jQBoia7KuzELsBCVxE:1709972303196",
"path": "/home/user/Desktop/subtitles_script.py"
}
]
}
},
{
"type": "command",
"parameters": {
"command": "echo password | sudo -S pip install pysrt",
"shell" :"true"
}
}
],
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "python /home/user/Desktop/subtitles_script.py",
"shell" :"true"
},
"expected": {
"type": "rule",
"rules":{
"expected": "true\n"
}
}
}
}

62
evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json Normal file
View File

@@ -0,0 +1,62 @@
{
"id": "a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a",
"snapshot": "multiapps",
"instruction": "I have developed a new Chrome extension myself, so it needs to be installed manually. Please help me install the extension located in the Desktop directory into the Chrome browser.",
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
"config": [
{
"type": "download",
"parameters": {
"files":[
{
"url":"https://drive.google.com/uc?export=download&id=1fe-MuG57hf12cEi4qI4dEI3OYJOHYoY2",
"path":"/home/user/Desktop/helloExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S apt-get update -y && echo password | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os"
],
"evaluator": {
"func": "is_in_list",
"result": {
"type": "find_unpacked_extension_path"
},
"expected": {
"type": "rule",
"rules": {
"expected": "/home/user/Desktop/helloExtension"
}
}
}
}

14
evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json
View File

@@ -23,6 +23,12 @@
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "download",
"parameters": {
@@ -75,10 +81,10 @@
"Liked Authors"
],
"urls": [
"https://jimfan.me/",
"https://ai.stanford.edu/~dahuang/",
"https://yukezhu.me/",
"https://www.eas.caltech.edu/people/anima"
["https://jimfan.me/", "https://research.nvidia.com/person/linxi-jim-fan"],
["https://research.nvidia.com/person/de-an-huang", "https://ai.stanford.edu/~dahuang/"],
["https://yukezhu.me/", "https://www.cs.utexas.edu/people/faculty-researchers/yuke-zhu", "https://experts.utexas.edu/yuke_zhu", "https://research.nvidia.com/person/yuke-zhu"],
["http://tensorlab.cms.caltech.edu/users/anima/", "https://www.eas.caltech.edu/people/anima"]
]
}
}

69
evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108",
"snapshot": "gimp",
"instruction": "I have a background image named 'anmi.png'. I need the image to be enhanced for better sharpness, as 'anmi_sharper.png'. Next, please use it as the background image for my 'index.html' page.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1BAZvR0BflPh70ClL2bHrbJ-xyV-nNQ1o",
"path": "/home/user/Desktop/anmi.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1zcyY7y_deKStc-AtU8pFFrJItOuyfrNK",
"path": "/home/user/Desktop/index.html"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/anmi.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"check_sharper",
"check_html_background_image"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/anmi_sharper.png",
"dest": "anmi_sharper.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/index.html",
"dest": "index.html"
}
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/anmi.png",
"dest": "anmi.png"
},
{
"type": "rule",
"rules": {
"type:": "value",
"value": "anmi_sharper.png"
}
}
]
}
}

80
evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json Normal file
View File

@@ -0,0 +1,80 @@
{
"id": "ce2b64a2-ddc1-4f91-8c7d-a88be7121aac",
"snapshot": "multiapps",
"instruction": "There are several pictures of mountains in my Pictures directory, but I don’t know the names of these mountains. Please help me identify these pictures and change the names of these pictures to the names of the mountains in the pictures.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sKdxRx4Uxs-wxohYkmxPfxOUpw72BINV&export=download&authuser=0&confirm=t&uuid=2cd18b66-93a6-4771-a92b-a1b1f0adf12d&at=APZUnTUvT9iZ0IdRnnNf1ys4mZaE:1709972639977",
"path": "/home/user/Pictures/picture1.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1iquZNI4ktx2915srQ1MMlhcwCuXnmq5T&export=download&authuser=0&confirm=t&uuid=e0bfd86f-a0f1-4a9e-8344-0f645eb379dd&at=APZUnTWQnCd7bNtf8ZN4KfkQUaJ9:1709972645590",
"path": "/home/user/Pictures/picture2.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ie3rhHzsW293xhnmgeZm6wQj_uaFmPS7&export=download&authuser=0&confirm=t&uuid=633c8b0c-0d3f-4f07-a772-945148956244&at=APZUnTX6OIN5YfudwPVT3Mxntd9E:1709972650733",
"path": "/home/user/Pictures/picture3.jpg"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os", "image"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1eWGQ3z-Ps5p784NZjj9KyhHfX7NbHgWd&export=download&authuser=0&confirm=t&uuid=8ba167f4-8669-477b-89ee-a8f6996f43d7&at=APZUnTWShYYueBb8hRnrjjabU1ZD:1709972787510",
"path": "/home/user/Desktop/image_script.py"
}
]
}
}
],
"func": "check_direct_json_object",
"result":{
"type": "vm_command_line",
"command":"python /home/user/Desktop/image_script.py",
"shell": "true"
},
"expected": {
"type": "rule",
"rules":{
"expected":{
"ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": "Kilimanjaro",
"999957c8de835bfa420d198270e7a6b079ee20ff53a3f214491e8134768a7c0b": "Himalayas",
"79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": "Hua"
},
"expect_in_result": true
}
}
}
}

51
evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "d68204bf-11c1-4b13-b48b-d303c73d4bf6",
"snapshot": "gimp",
"instruction": "Divide my image vertically into three equal sections with command line. Then rearrange them in order with a gradient of warm tones, progressively becoming warmer from left to right as a new image \"rearranged.png\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1CPGW_OZsfSWDdTU7CFrTjpzSAASyLy4w",
"path": "/home/user/Desktop/tilearray.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1aHwmnxL2CKEh_FhVpevY452-BQH2t5rG",
"path": "/home/user/Desktop/rearranged_gold.png"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_structure_sim",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/rearranged.png",
"dest": "rearranged.png"
},
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/rearranged_gold.png",
"dest": "rearranged_gold.png"
}
}
}

75
evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json Normal file
View File

@@ -0,0 +1,75 @@
{
"id": "da922383-bfa4-4cd3-bbad-6bebab3d7742",
"snapshot": "multiapps",
"instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs you are opening now in pdf format to /home/user/Documents/Blogs.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://lilianweng.github.io/posts/2023-06-23-agent/",
"https://lilianweng.github.io/posts/2024-02-05-human-data-quality/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fm4wqp91_3yzVSlenc9k2Tlh3-7QuRvu&export=download&authuser=0&confirm=t&uuid=5bb46f9e-ef04-4b52-bceb-a88f0058f696&at=APZUnTU6aWNzsiqLwAlyr_8pxKUd:1709971656456",
"path": "/home/user/Desktop/script.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "pip install PyMuPDF",
"shell": "true"
}
}
],
"func": "exact_match",
"result":{
"type": "vm_command_line",
"command":"python /home/user/Desktop/script.py",
"shell": "true"
},
"expected": {
"type": "rule",
"rules":{
"expected": "[1, 1]\n"
}
}
}
}

53
evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json Normal file
View File

@@ -0,0 +1,53 @@
{
"id": "dd60633f-2c72-42ba-8547-6f2c8cb0fdb0",
"snapshot": "multiapps",
"instruction": "I ran some simple code demos on the currently open google colab, and I think the effect is pretty good. Please help me extract the code in all code boxes, merge it into a \"task.py\" file and store it in the local Home directory.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://colab.research.google.com/github/stanfordnlp/dspy/blob/main/intro.ipynb#scrollTo=L1BHAoL_LRd7"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os", "writer"
],
"evaluator": {
"func": "compare_python_pure_text",
"result": {
"type": "vm_file",
"path": "/home/user/colab.py",
"dest": "colab.py"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1r0VyQZf5Hi8n8R9w9iGKMsqQaxAjYN7O&export=download&authuser=0&confirm=t&uuid=56efd2ff-4494-4026-9e96-82783b5ed600&at=APZUnTUL7BOMtV3FIoc0kXe18LWz:1709972550788",
"dest": "colab_Gold.py"
}
}
}

67
evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json Normal file
View File

@@ -0,0 +1,67 @@
{
"id": "e8172110-ec08-421b-a6f5-842e6451911f",
"snapshot": "gimp",
"instruction": "Open 'character.png' in GIMP and extract the pixel art character. Save the selected character as 'character_gimp.png'. Additionally, write a Python script to automate this selection process, ensuring it precisely mimics the manual extraction done in GIMP. Output the result from the script as 'character_code.png'.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1Tf7MvqBMHuIhvKmk-Ryr_qmzmkACFjxB",
"path": "/home/user/Desktop/character.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1wKG5X6LaN0tShsAK4lFg3OyFg8OGYYZg",
"path": "/home/user/Desktop/character_no_background_gold.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/character.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"check_structure_sim",
"check_structure_sim"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/character_gimp.png",
"dest": "character_gimp.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/character_code.png",
"dest": "character_code.png"
}
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/character_no_background_gold.png",
"dest": "character_no_background_gold.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/character_no_background_gold.png",
"dest": "character_no_background_gold.png"
}
]
}
}

1
evaluation_examples/settings/googledrive/credentials.json Normal file
View File

@@ -0,0 +1 @@
{"access_token": "ya29.a0Ad52N3969wUkQepy6SBOSw9Gjg4-MNPfEUBD3OZpajVfs9wL4DbfImk-5XawHjBkTdCKKBqG5R9XIX6KvvUzQDfB2BwVwb0MfLfLJDLALia7MRdPn4j6GAES372u3bSqJNNPMwVZA9j-THb3o5svJiKcJgwcoFKeKC_xaCgYKAScSARISFQHGX2MioJPeGh_8OM6z1_BujwRe3Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0eVpYfdSAjvbCCgYIARAAGA4SNwF-L9IrAgL6KVceiEVTjtQdmPki2I3m8ejP3lzTLL2Wa3-rdrYfU7eYeKDVCS5KRxa_xCE_pPY", "token_expiry": "2024-03-08T17:16:15Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0Ad52N3969wUkQepy6SBOSw9Gjg4-MNPfEUBD3OZpajVfs9wL4DbfImk-5XawHjBkTdCKKBqG5R9XIX6KvvUzQDfB2BwVwb0MfLfLJDLALia7MRdPn4j6GAES372u3bSqJNNPMwVZA9j-THb3o5svJiKcJgwcoFKeKC_xaCgYKAScSARISFQHGX2MioJPeGh_8OM6z1_BujwRe3Q0171", "expires_in": 3599, "refresh_token": "1//0eVpYfdSAjvbCCgYIARAAGA4SNwF-L9IrAgL6KVceiEVTjtQdmPki2I3m8ejP3lzTLL2Wa3-rdrYfU7eYeKDVCS5KRxa_xCE_pPY", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}

2
requirements.txt
View File

@@ -47,3 +47,5 @@ mutagen
easyocr
borb
pypdf2
pdfplumber