lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix multi apps

Browse Source
This commit is contained in:
tsuky_chen
2024-03-09 18:48:17 +08:00
parent 3761de4a05
commit f4ec36bdfb
15 changed files with 107 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
desktop_env/evaluators/metrics/vscode.py
View File

@@ -104,6 +104,27 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
return 0.0
import zipfile
from difflib import SequenceMatcher
import PyPDF2
def compare_pdf_content(content1, content2, text_similarity_threshold):
def extract_text_from_pdf(content):
with open("temp.pdf", "wb") as temp_pdf:
temp_pdf.write(content)
with open("temp.pdf", "rb") as temp_pdf:
pdf_reader = PyPDF2.PdfReader(temp_pdf)
text = ''
for page_num in range(len(pdf_reader.pages)):
page = pdf_reader.pages[page_num]
text += page.extract_text()
return text
text1 = extract_text_from_pdf(content1)
text2 = extract_text_from_pdf(content2)
similarity_ratio = SequenceMatcher(None, text1, text2).ratio()
return similarity_ratio >= text_similarity_threshold
def compare_zip_files(actual: str, expected: str, **options) -> float:
"""
@@ -128,7 +149,12 @@ def compare_zip_files(actual: str, expected: str, **options) -> float:
content1 = zip_file1.read(file_name)
content2 = zip_file2.read(file_name)
if content1 != content2:
if file_name.lower().endswith('.pdf'):
if compare_pdf_content(content1, content2, 0.95):
continue
else:
return 0.0
elif content1 != content2:
return 0.0
return 1.0

2
evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json
View File

@@ -54,7 +54,7 @@
"type": "rule",
"rules": {
"type": "text",
"text": "$ ls\n"
"text": " Ls"
}
}
}

2
evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json
View File

@@ -37,7 +37,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803",
"dest": "pic.jpg"
"dest": "pic_Gold.jpg"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/20236825-b5df-46e7-89bf-62e1d640a897.json
View File

@@ -47,7 +47,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1g2Trt9oxQyW_sx8aIztFA0zNsE4yNw2x&export=download&authuser=0&confirm=t&uuid=342751c4-54f1-4760-9326-e7388845ded0&at=APZUnTV5BcbaxIZrDglWbs84Oxln:1709623697315",
"dest": "res.txt"
"dest": "res_Gold.txt"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/227d2f97-562b-4ccb-ae47-a5ec9e142fbb.json
View File

@@ -63,7 +63,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=11kWQc1XFEqcIMuW0-NnZRSdv1199OmVI&export=download&authuser=0&confirm=t&uuid=694676fd-1ac9-4501-8acf-f48018494c7f&at=APZUnTV-koL51ka5dHum_HpGywv_:1709618406292",
"dest": "image.docx"
"dest": "image_Gold.docx"
}
}
}

2
evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json
View File

@@ -1,7 +1,7 @@
{
"id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
"snapshot": "libreoffice_impress",
"instruction": "I've noticed that the image on the second slide is too dim. Can you please enhance its brightness for me? Save the adjusted image on the Desktop and name it \"background\". Thank you!",
"instruction": "I've noticed that the image on the second slide is too dim. Can you please enhance its brightness for me? Save the adjusted image on the Desktop and name it \"background.png\". Thank you!",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{

4
evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json
View File

@@ -61,12 +61,12 @@
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1kAp7ulAR_h4snny212yg1xyR1cMy3H2Q&export=download&authuser=0&confirm=t&uuid=3f6cb74c-63cc-4653-9083-00626ef2fc11&at=APZUnTWuXvVM2w1Q9h0hOsuX6thn:1709789680904",
"dest": "paper01.pdf"
"dest": "paper01_Gold.pdf"
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1UMimItX51tzNXkIMGPpHOdPNF5Dx0Tpy&export=download&authuser=0&confirm=t&uuid=97b668a6-2d0d-4389-ac5e-234e931b4328&at=APZUnTVvuvbAE8r7jpK8AkzGUzyw:1709790384938",
"dest": "ans.docx"
"dest": "ans_Gold.docx"
}
],
"result": [

2
evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json
View File

@@ -63,7 +63,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1KbdlQC0qSAYewG8QnZgParnSwv3s3dub&export=download&authuser=0&confirm=t&uuid=15dcc25c-8168-425e-96e1-fd27e0d6904b&at=APZUnTVho4ZrREHf9DC4rKwdIi3R:1709557117932",
"dest": "price.docx"
"dest": "price_Gold.docx"
}
}
}

2
evaluation_examples/examples/multi_apps/8df7e444-8e06-4f93-8a1a-c5c974269d82.json
View File

@@ -36,7 +36,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1IKRu-dMFP4Aqzq5-4TOmOWVci0qvC27K&export=download&authuser=0&confirm=t&uuid=e2dabad2-5648-4bc3-a40f-f008089cd613&at=APZUnTVh5JD5nT3EvutwHIaSnJAT:1709633945616",
"dest": "Recruitment_and_retention_of_health_professionals_across_Europe.zip"
"dest": "Recruitment_and_retention_of_health_professionals_across_Europe_Gold.zip"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/aad10cd7-9337-4b62-b704-a857848cedf2.json
View File

@@ -59,7 +59,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1r2KJv0H3foo1WlWnArxdXnaew-yixNqL&export=download&authuser=0&confirm=t&uuid=633cc27c-d38b-4c45-907d-025341b4af1c&at=APZUnTV8AW5F_aLVooprdfgt-Q-Z:1709547335200",
"dest": "notes.docx"
"dest": "notes_Gold.docx"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json
View File

@@ -1,7 +1,7 @@
{
"id": "b337d106-053f-4d37-8da0-7f9c4043a66b",
"snapshot": "os",
"instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display relative line numbers. Please search the internet for a tutorial on enabling relative line numbers and setting it as the default for my local Vim.",
"instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the internet for a tutorial on adding line numbers in Vim and setting it as default for my local Vim.",
"source": "authors",
"config": [
{

4
evaluation_examples/examples/multi_apps/bb83cab4-e5c7-42c7-a67b-e46068032b86.json
View File

@@ -54,7 +54,7 @@
}
}
],
"func": "compare_docx_tables",
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/script.docx",
@@ -63,7 +63,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-Sol1W8S7Ybj-3KBJJarbcYUqS5wAQ1C&export=download&authuser=0&confirm=t&uuid=d967f546-b8f7-4ac2-b8fc-b1635f1cfbc4&at=APZUnTUazmbS2X3BSXDEQtJgobgf:1709559012053",
"dest": "script.docx"
"dest": "script_Gold.docx"
}
}
}

93
evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json
View File

@@ -7,7 +7,11 @@
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Departments/finance"]
"command": [
"mkdir",
"-p",
"/home/user/Documents/Departments/finance"
]
}
},
{
@@ -29,37 +33,52 @@
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": ["nautilus", "/home/user/Documents/Departments/finance"]
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"nautilus",
"/home/user/Documents/Departments/finance"
]
}
}
],
"trajectory": "trajectories/f5c13cdd-205c-4719-a562-348ae5cd1d91",
"related_apps": ["thunderbird", "os", "libreoffice_calc"],
"related_apps": [
"thunderbird",
"os",
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 10
}
}
],
"func": "check_accessibility_tree",
"result": {
"type": "accessibility_tree"
@@ -67,11 +86,27 @@
"expected": {
"type": "rule",
"rules": [
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"]},
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"]},
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"]},
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"]}
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"
]
},
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"
]
},
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"
]
},
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"
]
}
]
}
}
}
}

2
evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -26,7 +26,7 @@
}
}
],
"trajectory": "trajectories/",
"trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef",
"related_apps": [
"vscode",
"os"

3
requirements.txt
View File

@@ -44,5 +44,6 @@ dashscope
google-generativeai
PyYaml
mutagen
pytesseract
easyocr
borb
pypdf2