lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:ztjhz/DesktopEnv

Browse Source
This commit is contained in:
Siheng Zhao
2024-01-31 10:37:31 +08:00
parent cffed446fc 0e5eefbb86
commit 43a752c3e6
29 changed files with 1002 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
desktop_env/evaluators/README.md
View File

@@ -1,10 +1,20 @@
# Setup Instructions
## LibreOffice
For LibreOffice, please enter into the app first, and then enable the no pop-up when 'ctrl + s'.
## LibreOffice Press
### Setting Up the python-pptx Library
```shell
pip install python-pptx
```
## LibreOffice Writer
### Setting Up the python-docx Library
### Setting Up the python-docx and odfpy Library
```shell
pip install python-docx
pip install odfpy
```
## LibreOffice Calc

2
desktop_env/evaluators/getters/__init__.py
View File

@@ -21,5 +21,5 @@ from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config
from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player
from .vscode import get_vscode_config

28
desktop_env/evaluators/getters/vlc.py
View File

@@ -1,7 +1,8 @@
import logging
import os
from typing import Dict
from collections import Counter
from .general import get_vm_command_line
import requests
logger = logging.getLogger("desktopenv.getters.vlc")
@@ -58,3 +59,28 @@ def get_vlc_config(env, config: Dict[str, str]):
f.write(content)
return _path
def get_default_video_player(env, config: dict):
""" Gets the default application for a category or file extension.
"""
os_type = env.vm_platform
if os_type == "Linux":
extensions = ['3gp', '3gp', '3gpp', '3gpp', '3gpp2', '3gpp2', 'avi', 'avi', 'divx', 'divx', 'dv', 'dv', 'fli', 'fli', 'flv', 'flv', 'mp2t', 'mp2t', 'mp4', 'mp4', 'mp4v-es', 'mp4v-es', 'mpeg', 'mpeg', 'mpeg-system', 'mpeg-system', 'msvideo', 'msvideo', 'ogg', 'ogg', 'quicktime', 'quicktime', 'vnd.divx', 'vnd.divx', 'vnd.mpegurl', 'vnd.mpegurl', 'vnd.rn-realvideo', 'vnd.rn-realvideo', 'webm', 'webm', 'x-anim', 'x-anim', 'x-avi', 'x-avi', 'x-flc', 'x-flc', 'x-fli', 'x-fli', 'x-flv', 'x-flv', 'x-m4v', 'x-m4v', 'x-matroska', 'x-matroska', 'x-mpeg', 'x-mpeg', 'x-mpeg-system', 'x-mpeg-system', 'x-mpeg2', 'x-mpeg2', 'x-ms-asf', 'x-ms-asf', 'x-ms-asf-plugin', 'x-ms-asf-plugin', 'x-ms-asx', 'x-ms-asx', 'x-ms-wm', 'x-ms-wm', 'x-ms-wmv', 'x-ms-wmv', 'x-ms-wmx', 'x-ms-wmx', 'x-ms-wvx', 'x-ms-wvx', 'x-msvideo', 'x-msvideo', 'x-nsv', 'x-nsv', 'x-ogm', 'x-ogm', 'x-ogm+ogg', 'x-theora', 'x-theora', 'x-theora+ogg', 'x-theora+ogg']
apps = []
for ext in extensions:
app = get_vm_command_line(env, {"command": ["xdg-mime", "query", "default", f"video/{ext}"]})
if app:
apps.append(app)
if len(apps) == 0:
return 'unknown'
else:
return Counter(apps).most_common(1)[0][0]
elif os_type == "Darwin":
raise Exception("Unsupported operating system", os_type)
elif os_type == "Windows":
raise Exception("Unsupported operating system", os_type)
else:
raise Exception("Unsupported operating system", os_type)

3
desktop_env/evaluators/getters/vscode.py
View File

@@ -1,6 +1,6 @@
import logging
from typing import Any, Dict
import time
from .file import get_vm_file
from .replay import get_replay
@@ -27,6 +27,7 @@ def get_vscode_config(env, config: Dict[str, Any]) -> str:
]
get_replay(env, trajectory)
time.sleep(1.0)
return get_vm_file(env, {
"path": config["path"],

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -66,7 +66,7 @@ from .gimp import (
check_triangle_position,
check_structure_sim,
check_config_status,
compare_images,
compare_image_list,
increase_saturation,
decrease_brightness,
check_file_exists,

8
desktop_env/evaluators/metrics/general.py
View File

@@ -17,10 +17,16 @@ from .utils import _match_record, _match_value_to_rule
def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
if result is None:
return 0.
print(result, rules)
include = rules.get("include", [])
exclude = rules.get("exclude", [])
return all(r in result for r in include) and all(r not in result for r in exclude)
if all(r in result for r in include) and all(r not in result for r in exclude):
return 1.
else:
return 0.
def exact_match(result, rules) -> float:

65
desktop_env/evaluators/metrics/gimp.py
View File

@@ -4,7 +4,7 @@ from skimage.metrics import structural_similarity as ssim
from PIL import Image, ImageChops, ImageStat
def compare_images(pred_img_path_list: Union[str, List[str]],
def compare_image_list(pred_img_path_list: Union[str, List[str]],
gold_img_path_list: Union[str, List[str]]) -> float:
""" Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0
"""
@@ -197,6 +197,9 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
Check the brightness of src is lower than tgt and the structures are similar
gimp:7a4deb26-d57d-4ea9-9a73-630f66a7b568
"""
if src_path is None or tgt_path is None:
return 0.
img_src = Image.open(src_path)
img_tgt = Image.open(tgt_path)
@@ -222,6 +225,9 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path):
Check the saturation of src is higher than tgt and the structures are similar
gimp:554785e9-4523-4e7a-b8e1-8016f565f56a
"""
if src_path is None or tgt_path is None:
return 0.
img_src = Image.open(src_path)
hsv_img_src = img_src.convert('HSV')
img_tgt = Image.open(tgt_path)
@@ -254,17 +260,23 @@ def check_file_exists_and_structure_sim(src_path, tgt_path):
Check if the image has been exported to the desktop
gimp:77b8ab4d-994f-43ac-8930-8ca087d7c4b4
"""
if src_path is None or tgt_path is None:
return 0.
# Check if the file exists
export_file_exists = os.path.isfile(src_path)
if not export_file_exists:
return False
return 0.
# Check whether the target image is the same as the source image
img_src = Image.open(src_path)
img_tgt = Image.open(tgt_path)
structure_same = structure_check_by_ssim(img_src, img_tgt)
return structure_same
if structure_same:
return 1.
else:
return 0.
def check_triangle_position(tgt_path):
@@ -272,18 +284,20 @@ def check_triangle_position(tgt_path):
Check if the triangle is in the middle of the image.
gimp:f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce
"""
if tgt_path is None:
return 0.
# Load the image
img = Image.open(tgt_path)
img_array = np.array(img)
# We will determine if the triangle is in the middle of the picture by checking the centroid
# We assume the triangle is a different color from the background
# Find the unique colors
unique_colors = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0)
unique_colors, counts = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0, return_counts=True)
unique_colors_sorted = unique_colors[np.argsort(counts)]
# Assuming the background is the most common color and the triangle is a different color,
# we identify the triangle's color as the least common one
triangle_color = unique_colors[-1]
# Assuming the background is the most common color and the triangle is a different color
triangle_color = unique_colors_sorted[1]
# Create a mask where the triangle pixels are True
triangle_mask = np.all(img_array == triangle_color, axis=2)
@@ -323,6 +337,9 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path):
Check if the src image has higher contrast than the tgt image and the structures are similar
gimp:f723c744-e62c-4ae6-98d1-750d3cd7d79d
"""
if src_path is None or tgt_path is None:
return 0.
# Load images
source_image = Image.open(src_path)
target_image = Image.open(tgt_path)
@@ -345,6 +362,9 @@ def check_config_status(actual_config_path, rule):
"""
Check if the GIMP status is as expected
"""
if actual_config_path is None:
return 0.
with open(actual_config_path, 'r') as f:
content = f.readlines()
@@ -368,6 +388,10 @@ def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=Non
Check if the size of the src image is correct and the structure of the two images are similar.
gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15
"""
if src_path is None or tgt_path is None:
return 0.
# Load images
source_image = Image.open(src_path)
target_image = Image.open(tgt_path)
@@ -397,6 +421,9 @@ def check_palette_and_structure_sim(src_path, tgt_path):
Check if the src image is palette-based and the structure of the two images are similar
gimp:06ca5602-62ca-47f6-ad4f-da151cde54cc
"""
if src_path is None or tgt_path is None:
return 0.
# Check if the source image is palette-based
source_image = Image.open(src_path)
palette_based = source_image.mode == 'P'
@@ -416,6 +443,9 @@ def check_textbox_on_leftside(src_path):
Check if the textbox is on the left side of the image.
gimp:e2dd0213-26db-4349-abe5-d5667bfd725c
"""
if src_path is None:
return 0.
source_image = Image.open(src_path)
gray_image = source_image.convert("L")
width, height = source_image.size
@@ -441,6 +471,9 @@ def check_image_mirror(src_path, tgt_path):
Check if the image is mirrored
gimp:72f83cdc-bf76-4531-9a1b-eb893a13f8aa
"""
if src_path is None or tgt_path is None:
return 0.
# Load images
source_image = Image.open(src_path)
target_image = Image.open(tgt_path)
@@ -449,7 +482,10 @@ def check_image_mirror(src_path, tgt_path):
transposed_image = source_image.transpose(Image.FLIP_LEFT_RIGHT)
# Use 0.99 because the image may not be exactly mirrored by gimp
mirrored = structure_check_by_ssim(transposed_image, target_image, 0.99)
return mirrored
if mirrored:
return 1.
else:
return 0.
def check_green_background(src_path, tgt_path):
@@ -457,6 +493,9 @@ def check_green_background(src_path, tgt_path):
Check if the background of the source image is green.
gimp:734d6579-c07d-47a8-9ae2-13339795476b
"""
if src_path is None or tgt_path is None:
return 0.
# Load images
source_image = Image.open(src_path)
target_image = Image.open(tgt_path)
@@ -472,9 +511,9 @@ def check_green_background(src_path, tgt_path):
# Here, "green" means more green than red or blue
r, g, b = source_pixels[x, y][:3]
if not (g > r and g > b):
return False
return 0.
return True
return 1.
if __name__ == "__main__":
@@ -502,3 +541,7 @@ if __name__ == "__main__":
src_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/green_background_with_object.png"
tgt_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/white_background_with_object.png"
print(check_green_background(src_path, tgt_path))
tgt_path = "../../../cache/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce/Triangle_In_The_Middle.png"
print(check_triangle_position(tgt_path))

22
desktop_env/evaluators/metrics/slides.py
View File

@@ -267,19 +267,17 @@ def check_left_panel(accessibility_tree):
root = ET.fromstring(accessibility_tree)
for root_pane in root.iter('root-pane'):
for split_pane in root_pane.iter('split-pane'):
for panel in split_pane.iter('panel'):
for scroll_panel in panel.iter('scroll-pane'):
for document_frame in scroll_panel.iter('document-frame'):
# Get the left panel
panel_name = document_frame.get("name")
# visible = scroll_bar.attrib.get(f"{{{namespaces['st']}}}visible")
if panel_name == "Slides View":
# Left panel is open
return 1.0
for panel in root_pane.iter('panel'):
for split_pane in panel.iter('split-pane'):
# Get the left panel
if split_pane.attrib.get("{{{}}}parentcoord".format(namespaces['cp'])) == "(0, 0)":
# Get the visible attribute
visible = split_pane.attrib.get("{{{}}}visible".format(namespaces['st']))
if visible:
# decide if it is left panel
return 1.
# Left panel is not open
return 0.0
return 0.
def check_transition(pptx_file, rules):

229
desktop_env/server/main.py
View File

@@ -9,18 +9,29 @@ from typing import List, Dict, Tuple
import Xlib
import lxml.etree
import pyatspi
import pyautogui
import requests
from PIL import Image
from Xlib import display, X
from flask import Flask, request, jsonify, send_file, abort # , send_from_directory
from lxml.etree import _Element
from pyatspi import Accessible, StateType, STATE_SHOWING
from pyatspi import Action as ATAction
from pyatspi import Component, Document
from pyatspi import Text as ATText
from pyatspi import Value as ATValue
platform_name: str = platform.system()
if platform_name=="Linux":
import pyatspi
from pyatspi import Accessible, StateType, STATE_SHOWING
from pyatspi import Action as ATAction
from pyatspi import Component, Document
from pyatspi import Text as ATText
from pyatspi import Value as ATValue
BaseWrapper = Any
elif platform_name=="Windows":
from pywinauto import Desktop
from pywinauto.base_wrapper import BaseWrapper
Accessible = Any
from pyxcursor import Xcursor
@@ -176,22 +187,25 @@ def get_terminal_output():
# raise NotImplementedError
return "Currently not implemented for platform {:}.".format(platform.platform()), 500
return jsonify({"output": output, "status": "success"})
except:
return jsonify({"output": None, "status": "error"})
except Exception as e:
logger.error("Failed to get terminal output. Error: %s", e)
return jsonify({"status": "error", "message": str(e)}), 500
_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
, "win": "uri:deskat:uia.windows.microsoft.org"
}
def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) -> _Element:
def _create_atspi_node(node: Accessible, depth: int = 0, flag: Optional[str] = None) -> _Element:
# function _create_atspi_node {{{ #
if node.getRoleName() == "document spreadsheet":
flag = "calc"
if node.getRoleName() == "application" and node.name=="Thunderbird":
@@ -380,6 +394,175 @@ def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None)
break
xml_node.append(_create_atspi_node(ch, depth+1, flag))
return xml_node
# }}} function _create_atspi_node #
def _create_pywinauto_node(node: BaseWrapper, depth: int = 0, flag: Optional[str] = None) -> _Element:
# function _create_pywinauto_node {{{ #
#element_info: ElementInfo = node.element_info
attribute_dict: Dict[str, Any] = {"name": node.element_info.name}
# States {{{ #
attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower()
attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower()
attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower()
if hasattr(node, "is_minimized"):
try:
attribute_dict["{{{:}}}minimized".format(_accessibility_ns_map["st"])] = str(node.is_minimized()).lower()
except:
pass
if hasattr(node, "is_maximized"):
try:
attribute_dict["{{{:}}}maximized".format(_accessibility_ns_map["st"])] = str(node.is_maximized()).lower()
except:
pass
if hasattr(node, "is_normal"):
try:
attribute_dict["{{{:}}}normal".format(_accessibility_ns_map["st"])] = str(node.is_normal()).lower()
except:
pass
if hasattr(node, "is_unicode"):
try:
attribute_dict["{{{:}}}unicode".format(_accessibility_ns_map["st"])] = str(node.is_unicode()).lower()
except:
pass
if hasattr(node, "is_collapsed"):
try:
attribute_dict["{{{:}}}collapsed".format(_accessibility_ns_map["st"])] = str(node.is_collapsed()).lower()
except:
pass
if hasattr(node, "is_checkable"):
try:
attribute_dict["{{{:}}}checkable".format(_accessibility_ns_map["st"])] = str(node.is_checkable()).lower()
except:
pass
if hasattr(node, "is_checked"):
try:
attribute_dict["{{{:}}}checked".format(_accessibility_ns_map["st"])] = str(node.is_checked()).lower()
except:
pass
if hasattr(node, "is_focused"):
try:
attribute_dict["{{{:}}}focused".format(_accessibility_ns_map["st"])] = str(node.is_focused()).lower()
except:
pass
if hasattr(node, "is_keyboard_focused"):
try:
attribute_dict["{{{:}}}keyboard_focused".format(_accessibility_ns_map["st"])] = str(node.is_keyboard_focused()).lower()
except:
pass
if hasattr(node, "is_selected"):
try:
attribute_dict["{{{:}}}selected".format(_accessibility_ns_map["st"])] = str(node.is_selected()).lower()
except:
pass
if hasattr(node, "is_selection_required"):
try:
attribute_dict["{{{:}}}selection_required".format(_accessibility_ns_map["st"])] = str(node.is_selection_required()).lower()
except:
pass
if hasattr(node, "is_pressable"):
try:
attribute_dict["{{{:}}}pressable".format(_accessibility_ns_map["st"])] = str(node.is_pressable()).lower()
except:
pass
if hasattr(node, "is_pressed"):
try:
attribute_dict["{{{:}}}pressed".format(_accessibility_ns_map["st"])] = str(node.is_pressed()).lower()
except:
pass
if hasattr(node, "is_expanded"):
try:
attribute_dict["{{{:}}}expanded".format(_accessibility_ns_map["st"])] = str(node.is_expanded()).lower()
except:
pass
if hasattr(node, "is_editable"):
try:
attribute_dict["{{{:}}}editable".format(_accessibility_ns_map["st"])] = str(node.is_editable()).lower()
except:
pass
# }}} States #
# Component {{{ #
rectangle = node.rectangle()
attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = "({:d}, {:d})".format(rectangle.left, rectangle.top)
attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = "({:d}, {:d})".format(rectangle.width(), rectangle.height())
# }}} Component #
# Text {{{ #
text: str = node.window_text()
if text==attribute_dict["name"]:
text = ""
#if hasattr(node, "texts"):
#texts: List[str] = node.texts()[1:]
#texts: Iterable[str] = map(lambda itm: itm if isinstance(itm, str) else "".join(itm), texts)
#text += "\n".join(texts)
#text = text.strip()
# }}} Text #
# Selection {{{ #
if hasattr(node, "select"):
attribute_dict["selection"] = "true"
# }}} Selection #
# Value {{{ #
if hasattr(node, "get_step"):
attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(node.get_step())
if hasattr(node, "value"):
attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(node.value())
if hasattr(node, "get_value"):
attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(node.get_value())
elif hasattr(node, "get_position"):
attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(node.get_position())
if hasattr(node, "min_value"):
attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(node.min_value())
elif hasattr(node, "get_range_min"):
attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(node.get_range_min())
if hasattr(node, "max_value"):
attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(node.max_value())
elif hasattr(node, "get_range_max"):
attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(node.get_range_max())
# }}} Value #
attribute_dict["{{{:}}}class".format(_accessibility_ns_map["win"])] = str(type(node))
node_role_name: str = node.class_name().lower().replace(" ", "-")
node_role_name = "".join( map( lambda ch: ch if ch.isidentifier()\
or ch in {"-"}\
or ch.isalnum()
else "-"
, node_role_name
)
)
if node_role_name.strip() == "":
node_role_name = "unknown"
xml_node = lxml.etree.Element(
node_role_name,
attrib=attribute_dict,
nsmap=_accessibility_ns_map
)
if text is not None and len(text)>0 and text!=attribute_dict["name"]:
xml_node.text = text
# HYPERPARAMETER
if depth==50:
logger.warning("Max depth reached")
#print("Max depth reached")
return xml_node
for i, ch in enumerate(node.children()):
# HYPERPARAMETER
if i>=2048:
logger.warning("Max width reached")
#print("Max width reached")
break
xml_node.append(_create_pywinauto_node(ch, depth+1, flag))
return xml_node
# }}} function _create_pywinauto_node #
@app.route("/accessibility", methods=["GET"])
def get_accessibility_tree():
@@ -391,7 +574,15 @@ def get_accessibility_tree():
desktop_xml: _Element = _create_atspi_node(desktop, 0)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
# TODO: Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned
elif os_name == "Windows":
# Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned
desktop: Desktop = Desktop(backend="uia")
xml_node = lxml.etree.Element("desktop", nsmap=_accessibility_ns_map)
for wnd in desktop.windows():
logger.debug("Win UIA AT parsing: %s(%d)", wnd.element_info.name, len(wnd.children()))
node: _Element = _create_pywinauto_node(wnd, 1)
xml_node.append(node)
return jsonify({"AT": lxml.etree.tostring(xml_node, encoding="unicode")})
else:
return "Currently not implemented for platform {:}.".format(platform.platform()), 500

76
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json Normal file
View File

@@ -0,0 +1,76 @@
{
"id": "01b269ae-2111-4a07-81fd-3fcd711993b0",
"snapshot": "libreoffice_calc",
"instruction": "Fill all the blank cells with the value in the cell above it",
"source": "https://www.youtube.com/shorts/VrUzPTIwQ04",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032",
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
}
}
],
"trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0",
"related_apps": [
"msoffice_excel"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Student_Level_Fill_Blank.xlsx - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916",
"dest": "Student_Level_Fill_Blank_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx",
"dest": "Student_Level_Fill_Blank.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

2
evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json
View File

@@ -1,7 +1,7 @@
{
"id": "455d3c66-7dc6-4537-a39a-36d3e9119df7",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me export an Impress file to a .jpg image file and save it as res.jpg on the Desktop? ",
"instruction": "Could you help me export an Impress file to a .png image file and save it as res.png on the Desktop? ",
"source": "https://stackoverflow.com/questions/75626383/how-export-libreoffice-impress-to-image",
"config": [
{

2
evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json
View File

@@ -19,7 +19,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')"
]
}
}

4
evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
"url": "https://drive.usercontent.google.com/download?id=1YhHNlRsL7lJBsqRYctz4CmEoD1g8oAm0&export=download&authuser=0&confirm=t&uuid=16776039-9eae-4ee7-ae0b-8b2d71cb25e1&at=APZUnTWVT6sfD3MQEADssAEc4Pwn:1706622286569",
"path": "Desktop/Novels_Intro_Packet.docx"
}
]
@@ -64,4 +64,4 @@
"dest": "Novels_Intro_Packet.docx"
}
}
}
}

59
evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json
View File

@@ -36,7 +36,8 @@
"parameters": {
"command": [
"gnome-terminal",
"--maximize"
"--maximize",
"--working-directory=/home/user/"
]
}
}
@@ -59,43 +60,27 @@
}
}
],
"func": ["check_include_exclude", "check_include_exclude"],
"conj": "and",
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi"
]
},
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi"
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"no libreoffice is running\"; else echo \"libreoffice is still running\"; fi; output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"not killed from terminal\"; else echo \"killed from terminal\"; fi"
]
},
"expected": {
"type": "rule",
"rules": {
"include": [
"no libreoffice is running",
"killed from terminal"
],
"exclude": [
"libreoffice is still running",
"not killed from terminal"
]
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"true\n"
]
}
},
{
"type": "rule",
"rules": {
"include": [
"true\n"
]
}
}
]
}
}
}

51
evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json
View File

@@ -6,15 +6,12 @@
"config": [
{
"type": "download",
"path": "",
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git"
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aANBCEHT6K8GmHDMEYtL1LQXdApqVoGv&export=download",
"path": "/home/user/Downloads/binder.zip"
}
]
}
},
@@ -24,7 +21,17 @@
"command": [
"/bin/bash",
"-c",
"mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md"
"git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main && git config receive.denyCurrentBranch ignore "
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"unzip -q /home/user/Downloads/binder.zip -d /home/user/projects/ && cd /home/user/projects/binder && git init --initial-branch main && git remote add origin /home/user/projects/remote_project"
]
}
},
@@ -34,7 +41,7 @@
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/projects/hello_world"
"--working-directory=/home/user/projects/binder"
]
}
}
@@ -45,19 +52,23 @@
"terminal"
],
"evaluator": {
"func": "compare_docx_files",
"func": "check_include_exclude",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/notes.docx",
"dest": "notes.docx"
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"cd /home/user/projects/remote_project; git log --oneline | head -n 1 | awk '{for (i=2; i<=NF; i++) {printf \"%s%s\", $i, (i==NF ? \"\" : \" \")}; print \"\"}'; remote_id=$(git log --oneline | head -n 1 | awk '{print $1}'); cd /home/user/projects/binder; local_id=$(git log --oneline | head -n 1 | awk '{print $1}'); if [ \"${local_id}\" = \"${remote_id}\" ]; then echo \"repo is synchronous\"; else echo \"repo is not synchronous\"; fi"
]
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download",
"dest": "notes_gold.docx"
},
"options": {
"ignore_blanks": true
"type": "rule",
"rules": {
"include": [
"daily update",
"repo is synchronous"
]
}
}
}
}

53
evaluation_examples/examples/multi_apps/2fe4b718-3bd7-46ec-bdce-b184f5653624.json Normal file
View File

@@ -0,0 +1,53 @@
{
"id": "2fe4b718-3bd7-46ec-bdce-b184f5653624",
"snapshot": "vlc",
"instruction": "Could you help me create an Animated GIF src_clip.gif from a video file using VLC and GIMP from the source of video \"src.mp4\" in the desktop, 5-second clip beginning at 00:03?",
"source": "https://www.maketecheasier.com/create-gif-from-video-gimp/",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"gimp"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1krQ_vN3QiboMttWrjDY-YROCWZq6cBtb&export=download",
"path": "/home/user/Desktop/src.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"vlc"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc",
"gimp"
],
"evaluator": {
"func": "compare_images",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/src_clip.gif",
"dest": "src_clip.gif"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1kDq8pakgdZvKh8CFpR5368stf14gwFCp&export=download",
"dest": "src_clip_gold.gif"
}
}
}

71
evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "3680a5ee-6870-426a-a997-eba929a0d25c",
"snapshot": "libreoffice_calc",
"instruction": "I have file1.xlsx and file2.ods on the Desktop and each has one column. Help me use only the command line to merge these two columns into one LibreOffice Calc file called output.csv and open it from terminal.",
"source": "https://unix.stackexchange.com/questions/510850/how-to-open-calc-from-terminal-and-insert-files",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ofMPSBPXtt0h6t0arjYo0GMtKAAhfEPz&export=download",
"path": "/home/user/Desktop/file1.xlsx"
},
{
"url": "https://drive.usercontent.google.com/download?id=15QOxr0_MiHR80ppMX02aHnnsWs25aXgZ&export=download",
"path": "/home/user/Desktop/file2.ods"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/Desktop"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"terminal"
],
"evaluator": {
"func": ["check_include_exclude", "compare_csv"],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(ps aux | grep \"[s]office\" | awk '{print $7}' | grep -E \"pts/|tty\"); if [ -z \"$output\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi;"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/output.csv",
"dest": "output.csv"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"use terminal"
]
}
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1OBQXIJ06HQeodaCwU7z0Kq_-d3YuGNqE&export=download",
"dest": "output_gold.csv"
}
]
}
}

6
evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -1,7 +1,7 @@
{
"id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
"snapshot": "chrome",
"instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
"instruction": "Help me export charts, graph or other images from docx files received in email \"Lecture Document\" in Notes folder and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
@@ -58,7 +58,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
@@ -93,7 +93,7 @@
"chrome"
],
"evaluator": {
"func": "compare_images",
"func": "compare_image_list",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",

116
evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json Normal file
View File

@@ -0,0 +1,116 @@
{
"id": "510f64c8-9bcc-4be1-8d30-638705850618",
"snapshot": "vscode",
"instruction": "Could you start VS Code in folder ~/Desktop/project from the terminal?",
"source": "https://www.geeksforgeeks.org/how-to-start-vs-code-from-the-terminal-command-line/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download",
"path": "/home/user/Downloads/vscodeEvalExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip && mkdir -p ~/Desktop/project/.vscode && history -c && echo > ~/.bash_history"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052",
"path": "/home/user/Desktop/project/main.py"
},
{
"url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290",
"path": "/home/user/Desktop/project/README.md"
},
{
"url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443",
"path": "/home/user/Desktop/project/.vscode/settings.json"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode",
"terminal"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"killall gnome-terminal-server"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"func": ["check_include_exclude", "compare_config"],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"use_terminal=$(cat ~/.bash_history | grep \"[c]ode \"); if [ -z \"$use_terminal\" ]; then echo \"false\"; else echo \"true\"; fi"
]
},
{
"type": "vscode_config",
"vscode_extension_command": "OpenProject",
"path": "/home/user/OpenProject.txt",
"dest": "OpenProject.txt"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"true"
]
}
},
{
"type": "rule",
"rules": {
"expect": "project"
}
}
]
}
}

93
evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json Normal file
View File

@@ -0,0 +1,93 @@
{
"id": "58565672-7bfe-48ab-b828-db349231de6b",
"snapshot": "chrome",
"instruction": "Can you assist me by opening the first link in the latest email in Bills folder from Thunderbird and displaying it in a new Chrome tab?",
"source": "https://superuser.com/questions/1792660/open-link-from-other-application-does-not-open-the-url-in-firefox",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.apple.com/",
"https://en.sjtu.edu.cn/",
"https://scholar.google.com/"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=88b71c06-b5b9-4108-a516-24c46fe9273d&at=APZUnTXK2q16yG7u43034ou5uCoA:1706600462917",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "is_expected_tabs",
"result": {
"type": "open_tabs_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"urls": [
"https://www.apple.com/",
"https://en.sjtu.edu.cn/",
"https://scholar.google.com/",
"https://www.amazon.com/"
]
}
}
}
}

26
evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json Normal file
View File

@@ -0,0 +1,26 @@
{
"id": "937087b6-f668-4ba6-9110-60682ee33441",
"snapshot": "vlc",
"instruction": "I am currently using a ubuntu system. Could you help me set the default video player as VLC?",
"source": "https://superuser.com/questions/187440/set-default-ubuntu-video-player-as-vlc",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"vlc",
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "default_video_player"
},
"expected": {
"type": "rule",
"rules": {
"include": [
"vlc.desktop"
]
}
}
}
}

6
evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json
View File

@@ -1,7 +1,7 @@
{
"id": "b52b40a5-ad70-4c53-b5b0-5650a8387052",
"snapshot": "chrome",
"instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
"instruction": "Could you help me merge all PDF files in the \"Paper Recommendation\" email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
@@ -58,7 +58,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"url": "https://drive.usercontent.google.com/download?id=1_SujjgvE6SkrIINB7bd-4quti7ICGaB_&export=download&authuser=0&confirm=t&uuid=aa95822d-55ca-46e0-8b52-14d495b41995&at=APZUnTX7f8cbJ00STW_kV8kMY1KW:1706609819112",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
@@ -104,7 +104,7 @@
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805",
"path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=de7b274a-25b8-4361-a614-4910a88ba4bd&at=APZUnTVRw21zrIY4ydp9sagr9ZhZ:1706609799766",
"dest": "attachment_full_gold.pdf"
}
}

95
evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json Normal file
View File

@@ -0,0 +1,95 @@
{
"id": "ee9a3c83-f437-4879-8918-be5efbb9fac7",
"snapshot": "libreoffice_calc",
"instruction": "Could you help me convert the opened ods file in the desktop to csv file with the same file name using command line when Libreoffice instance is running?",
"source": "https://stackoverflow.com/questions/64589140/convert-ods-to-csv-using-command-line-when-libreoffice-instance-is-running",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1V6e4KB6Sabp6nAHkq5Seun0qbjwEfdaF&export=download",
"path": "/home/user/Desktop/file_example_ODS_5000.ods"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/file_example_ODS_5000.ods"
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"history -c && echo > ~/.bash_history && sleep 3"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/Desktop"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"terminal"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"killall gnome-terminal-server"
]
}
}
],
"func": ["check_include_exclude", "compare_csv"],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"use_terminal=$(cat ~/.bash_history | grep \"\\(soffice\\|libreoffice\\).\\+--convert-to\\s\\+csv\"); if [ -z \"$use_terminal\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/file_example_ODS_5000.csv",
"dest": "file_example_ODS_5000.csv"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"use terminal"
]
}
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1L8cJjHYDtZymQp4-QdNZ9i6jzZzsXJm1&export=download",
"dest": "file_example_ODS_5000_gold.csv"
}
]
}
}

21
evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json
View File

@@ -1,18 +1,33 @@
{
"id": "5812b315-e7bd-4265-b51f-863c02174c28",
"snapshot": "os",
"instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".",
"instruction": "Please create an SSH user named \"charles\" with password \"Ex@mpleP@55w0rd!\" on Ubuntu who is only allowed to access the folder \"test1\".",
"source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": "mkdir /test1",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[[ $(getent passwd charles) && $(getent passwd charles | cut -d: -f6) = \"/test1\" ]] && echo \"User charles exists and has /test1 as their home directory\" || echo \"User charles does not exist or doesn't have /test1 as their home directory\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "User charles exists and has /test1 as their home directory"
}
}
}
}

23
evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json
View File

@@ -1,17 +1,34 @@
{
"id": "c56de254-a3ec-414e-81a6-83d2ce8c41fa",
"snapshot": "os",
"instruction": "Could you please help me to extract text from a non-indexed PDF document on the current system?",
"instruction": "I want to install the lxml package on my ubuntu system. Can you help me?",
"source": "https://superuser.com/questions/28426/how-to-extract-text-with-ocr-from-a-pdf-on-linux",
"config": [
{
"type": "execute",
"parameters": {
"command": "sudo apt-get update && sudo apt-get install -y python3",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "python3 -c \"import lxml\" >/dev/null 2>&1 && echo \"lxml is installed\" || echo \"lxml is not installed\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "lxml is installed"
}
}
}
}
}

22
evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json
View File

@@ -1,17 +1,33 @@
{
"id": "cc9d4f34-1ca0-4a1b-8ff2-09302696acb9",
"snapshot": "os",
"instruction": "I need you to execute a specific process or script from the terminal and ensure that it continues to run independently, even after the terminal session is terminated. This involves redirecting the process's output to a file and disassociating the process from the terminal session's control.",
"source": "https://superuser.com/questions/178587/how-do-i-detach-a-process-from-terminal-entirely",
"instruction": "Currently, the program my_process.sh is running in the system and I want to end the process. Can you help me?",
"source": "https://stackoverflow.com/questions/9346211/how-to-kill-a-process-on-a-port-on-ubuntu",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo -e '#!/bin/bash\n\nwhile true; do\necho \"Running my_process...\"\nsleep 1\ndone' > my_process.sh && chmod +x my_process.sh && nohup ./my_process.sh > /dev/null 2>&1 &",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[[ -z $(ps -ef | grep -v grep | grep my_process) ]] && echo \"Process killed\" || echo \"Process still running\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Process killed"
}
}
}
}

21
evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json
View File

@@ -4,6 +4,27 @@
"instruction": "Please help me use VS Code to open the \"project\" in the \"user\" folder under \"home\".",
"source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download",
"path": "/home/user/Downloads/vscodeEvalExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip"
]
}
},
{
"type": "launch",
"parameters": {

2
evaluation_examples/settings/googledrive/credentials.json
View File

@@ -1 +1 @@
{"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-29T05:13:41Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
{"access_token": "ya29.a0AfB_byBdrcgbmWKGyEUOxxuJBhxOs1uu0AqWeLgJKKKLG_dVg4iQKJAdiWD2oetHrKn17p4ZtfX-vt1VQ0BiF2MPD2exX1oESsQkXaO8q4TM1olIpadvlUBqUkqKJcjCqn1dp1oaTVYU-Srf2wQCGnDt3ozjljdkHXN_MQaCgYKAa4SARISFQHGX2MiWnixlrP3Se3vEV73_4fenA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-30T11:20:53Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byBdrcgbmWKGyEUOxxuJBhxOs1uu0AqWeLgJKKKLG_dVg4iQKJAdiWD2oetHrKn17p4ZtfX-vt1VQ0BiF2MPD2exX1oESsQkXaO8q4TM1olIpadvlUBqUkqKJcjCqn1dp1oaTVYU-Srf2wQCGnDt3ozjljdkHXN_MQaCgYKAa4SARISFQHGX2MiWnixlrP3Se3vEV73_4fenA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}

9
mm_agents/gpt_4v_agent.py
View File

@@ -45,11 +45,15 @@ def linearize_accessibility_tree(accessibility_tree):
linearized_accessibility_tree += node.attrib.get('name') + "\t"
if node.text:
linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t"
elif node.get("{uri:deskat:uia.windows.microsoft.org}class", "").endswith("EditWrapper")\
and node.get("{uri:deskat:value.at-spi.gnome.org}value"):
text: str = node.get("{uri:deskat:value.at-spi.gnome.org}value")
linearized_accessibility_tree += (text if '"' not in text else '"{:}"'.format(text.replace('"', '""'))) + "\t"
else:
linearized_accessibility_tree += '""\t'
linearized_accessibility_tree += node.attrib.get(
'{uri:deskat:component.at-spi.gnome.org}screencoord') + "\t"
linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size') + "\n"
'{uri:deskat:component.at-spi.gnome.org}screencoord', "") + "\t"
linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size', "") + "\n"
return linearized_accessibility_tree
@@ -278,6 +282,7 @@ class GPT4v_Agent:
elif self.exp in ["som", "seeact"]:
_screenshot = previous_obs["screenshot"]
_linearized_accessibility_tree = previous_obs["accessibility_tree"]
logger.debug("LINEAR AT: %s", _linearized_accessibility_tree)
messages.append({
"role": "user",