lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update multi apps

Browse Source
This commit is contained in:
tsuky_chen
2024-03-07 01:24:36 +08:00
parent 69ef653a7c
commit 5b5475094e
8 changed files with 215 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/evaluators/metrics/__init__.py
View File

@@ -47,7 +47,8 @@ from .docs import (
check_file_exists,
check_tabstops,
compare_contains_image,
compare_docx_images
compare_docx_images,
compare_image_text
)
from .general import (
check_csv,

11
desktop_env/evaluators/metrics/chrome.py
View File

@@ -91,6 +91,17 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float:
bookmark_bar_websites_urls = [bookmark['url'] for bookmark in bookmarks['bookmark_bar']['children'] if
bookmark['type'] == 'url']
return 1. if set(bookmark_bar_websites_urls) == set(rule['urls']) else 0.
elif rule['type'] == "liked_authors_websites_urls":
# Check if "liked authors" folder exists
liked_authors_folder = next((bookmark for bookmark in bookmarks['bookmark_bar']['children'] if
bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None)
if liked_authors_folder:
# Check if it contains the specified URLs
liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if
bookmark['type'] == 'url']
return 1. if set(liked_authors_urls) == set(rule['urls']) else 0.
else:
return 0.
else:
raise TypeError(f"{rule['type']} not support yet!")

10
desktop_env/evaluators/metrics/docs.py
View File

@@ -183,6 +183,16 @@ def compare_docx_images(docx_file1, docx_file2):
return 0
return 1
import pytesseract
def compare_image_text(image_path, rule):
img = Image.open(image_path)
img_text = pytesseract.image_to_string(img)
if rule['type'] == 'text':
return 1 if rule['text'] in img_text else 0
else:
raise ValueError("Unsupported rule type")
def compare_line_spacing(docx_file1, docx_file2):
if not compare_docx_files(docx_file1, docx_file2):
return 0

4
desktop_env/evaluators/metrics/slides.py
View File

@@ -182,7 +182,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
else:
return None
if get_slide_notes(slide1) != get_slide_notes(slide2) and examine_note:
if get_slide_notes(slide1).strip() != get_slide_notes(slide2).strip() and examine_note:
return 0
# check if the shapes are the same
for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
@@ -235,7 +235,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
return 0
if hasattr(shape1, "text") and hasattr(shape2, "text"):
if shape1.text != shape2.text and examine_text:
if shape1.text.strip() != shape2.text.strip() and examine_text:
return 0
# check if the paragraphs are the same

61
evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json Normal file
View File

@@ -0,0 +1,61 @@
{
"id": "02ce9a50-7af2-47ed-8596-af0c230501f8",
"snapshot": "libreoffice_writer",
"instruction": "I'm using libreoffice writer to write a tutorial about linux, and now I want to show the results obtained by using the \"ls\" command in /home/user. Please run this command and save the screenshot as 'ls.png' on Desktop.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1cOYh50n0S3-Q_h8HDTCEPv2v6NsJl07m&export=download&authuser=0&confirm=t&uuid=9741b52a-d998-4875-94ec-ab0a45a40bf9&at=APZUnTX__WVALT4i0EIQoRAWkm-k:1709715620836",
"path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('f11'); time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.scroll(-20)"
]
}
}
],
"trajectory": "trajectories/02ce9a50-7af2-47ed-8596-af0c230501f8",
"related_apps": [
"libreoffice_writer",
"os"
],
"evaluator": {
"func": "compare_image_text",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/ls.png",
"dest": "ls.png"
},
"expected": {
"type": "rule",
"rules": {
"type": "text",
"text": "$ ls\n"
}
}
}
}

48
evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json Normal file
View File

@@ -0,0 +1,48 @@
{
"id": "09a37c51-e625-49f4-a514-20a773797a8a",
"snapshot": "libreoffice_writer",
"instruction": "I received a request from my friend that he wanted me to help him modify a picture. On the Desktop is the requirement doc and the picture to be adjusted. Modify the image as he said and save modified pic as \"pic.jpg\" on Desktop. Thanks!",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1npPOtBdE5t_mzScyA94vDlxB8SCZNubv&export=download&authuser=0&confirm=t&uuid=64ee33b7-66a5-4f8e-9c98-95bea9521504&at=APZUnTW74Ntotdg-xRjaoxuDFju9:1709727407282",
"path": "/home/user/Desktop/requirment.docx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1eQHixX0TTExye0lbmHQny5CYhaREB5fQ&export=download&authuser=0&confirm=t&uuid=8546ef19-ea50-4e1a-9296-2db0302afd62&at=APZUnTVEmYOeAIdok_anTWOVHBCp:1709727527585",
"path": "/home/user/Desktop/ChMkKV8wsR6IBfEtABYfc0Tgu9cAAA1lQHO_78AFh-L733.jpg"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/requirment.docx"
}
}
],
"trajectory": "trajectories/09a37c51-e625-49f4-a514-20a773797a8a",
"related_apps": [
"libreoffice_writer",
"gimp",
"os"
],
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803",
"dest": "pic.jpg"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/pic.jpg",
"dest": "pic.jpg"
}
}
}

44
evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json Normal file
View File

@@ -0,0 +1,44 @@
{
"id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
"snapshot": "libreoffice_impress",
"instruction": "I found the image on the second slide is too dark. Could you make it brighter for me? Keep the modified picture on Desktop. Name it \"background\". Thanks!",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1SD5GlVUuG9RijMCEzuniPq1dIWxfr8xQ&export=download&authuser=0&confirm=t&uuid=67c13e54-9368-4e77-bda9-31776ec37479&at=APZUnTXGR14_2pztP7HoAiELiztJ:1709709598471",
"path": "/home/user/Desktop/PPT-Template_widescreen.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/PPT-Template_widescreen.pptx"
}
}
],
"trajectory": "trajectories/4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
"related_apps": [
"gimp",
"libreoffice_impress",
"os"
],
"evaluator": {
"func": "check_brightness_decrease_and_structure_sim",
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/background.png",
"dest": "background.png"
},
"result": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1lpRSXEoZq3ENOG5ekaAsBQSNv5ig0mDr&export=download&authuser=0&confirm=t&uuid=4cb10a33-81b3-4814-a969-f469832e33e5&at=APZUnTWN3pyiVpS003vLOgCcq2gu:1709710047375",
"dest": "image_original.png"
}
}
}

42
evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json
View File

@@ -4,6 +4,25 @@
"instruction": "I'm really enjoying this paper. Could you please find the personal webpages of the first author and the last three authors, and add them to a browser bookmark folder named 'Liked Authors'?",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "download",
"parameters": {
@@ -40,15 +59,28 @@
],
"trajectory": "trajectories/a82b78bb-7fde-4cb3-94a4-035baf10bcf0",
"related_apps": [
"chrome",
"pdf"
],
"evaluator": {
"postconfig": [],
"func": "",
"func": "is_expected_bookmarks",
"result": {
"type": "bookmarks"
},
"expected": {
},
"options": {
"type": "rule",
"rules": {
"type": "liked_authors_websites_urls",
"names": [
"Liked Authors"
],
"urls": [
"https://jimfan.me/",
"https://ai.stanford.edu/~dahuang/",
"https://yukezhu.me/",
"https://www.eas.caltech.edu/people/anima"
]
}
}
}
}
}