lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/xlang-ai/DesktopEnv

Browse Source
This commit is contained in:
rhythmcao
2024-03-02 21:09:37 +08:00
parent 2aff825945 2e9e3f355a
commit baf2a56ac8
7 changed files with 316 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
desktop_env/evaluators/metrics/docs.py
View File

@@ -13,6 +13,7 @@ from odf.text import P
from odf.text import Span
from skimage.color import deltaE_ciede2000
from skimage.color import rgb2lab
from fuzzywuzzy import fuzz
logger = logging.getLogger("desktopenv.metric.docs")
@@ -57,6 +58,8 @@ def contains_page_break(docx_file):
def compare_docx_files(file1, file2, **options):
ignore_blanks = options.get('ignore_blanks', True)
content_only = options.get('content_only', False)
def get_paragraph_texts_odt(document):
paragraphs = document.getElementsByType(P)
paragraph_texts = []
@@ -89,6 +92,13 @@ def compare_docx_files(file1, file2, **options):
print("Unsupported file types or mismatch between file types.")
return 0
if content_only:
# Compare the content of the documents
text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
similarity = fuzz.ratio(text1, text2) / 100.0
return similarity
# Process and compare documents
if ignore_blanks:
text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()

17
desktop_env/evaluators/metrics/vscode.py
View File

@@ -1,5 +1,7 @@
import copy
import json
from typing import Dict
import json, copy
def check_json_keybindings(actual: str, expected: str, **options) -> float:
"""
@@ -10,6 +12,7 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float:
Return:
float: the score
"""
def direct_load_json(fp):
try:
with open(fp, 'r') as f:
@@ -17,7 +20,7 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float:
return data
except:
return None
def skip_first_line_load_json(fp):
try:
with open(fp, 'r') as f:
@@ -54,7 +57,7 @@ def check_json_settings(actual: str, expected: str, **options) -> float:
with open(actual, 'r') as f:
data = json.load(f)
expect = expected['expected']
data_copy = copy.deepcopy(data)
data_copy.update(expect)
@@ -128,3 +131,11 @@ def is_extension_installed(actual: str, rules: Dict, **options):
return 0.0
else:
raise NotImplementedError
def check_python_file_by_test_suite(actual_file, test_suites, **options) -> float:
pass
def check_python_file_by_gold_file(actual_file, gold_file: str, **options) -> float:
pass

67
evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json
View File

@@ -1,9 +1,48 @@
{
"id": "0c825995-5b70-4526-b663-113f4c999dd2",
"snapshot": "libreoffice_calc",
"instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)'? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!",
"instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)' under /environment_policy folder? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!",
"source": "authors",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'environment_policy_report (draft).doc' or title = 'environment_policy_report (draft).docx' or title = 'environment_policy_report (draft)'",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "command",
"parameters": {
@@ -30,16 +69,26 @@
"os"
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
"func": "compare_docx_files",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"path_list": [
[
"environment_policy_report (draft).docx"
]
],
"dest": [
"environment_policy_report (draft).docx"
]
},
"expected": {
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1A2ti9JncAfIa6ks7FTJWHtYlZo-68FtM&export=download",
"dest": "environment_policy_report (draft)_gold.docx"
},
"options": {
"options": {
"content_only": true
}
}
}

2
evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json
View File

@@ -49,6 +49,8 @@
],
"trajectory": "trajectories/26150609-0da3-4a7d-8868-0faf9c5f01bb",
"related_apps": [
"vs_code",
"os"
],
"evaluator": {
"postconfig": [],

186
evaluation_examples/examples/multi_apps/881deb30-9549-4583-a841-8270c65f2a17.json Normal file
View File

@@ -0,0 +1,186 @@
{
"id": "881deb30-9549-4583-a841-8270c65f2a17",
"snapshot": "libreoffice_calc",
"instruction": "I want to find a faculty job in Hong Kong, so I am more curious about the \"Early Career Scheme\" of those schools is better to apply, please help me to count all the documents in the ecs pdf files in my hand, and organize the pass rate of each school by year into table!",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/Documents/Fundings/ecs",
"/home/user/Documents/Fundings/grf"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Documents/Fundings/ecs/ecs15.pdf",
"url": "https://drive.google.com/uc?id=1FTiT3mLlkehe2yWVdSMWr1w4ltLtQZUy&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs16.pdf",
"url": "https://drive.google.com/uc?id=1DMzZyhDey3lDsQ7fcPiJm9AYGECZms3q&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs17.pdf",
"url": "https://drive.google.com/uc?id=1TgAWk7FiV8fNrG2L3-Eu7BDccqtXebXY&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs23.pdf",
"url": "https://drive.google.com/uc?id=11DVxH4eRjECUxZNbUhYNJhgD0Y5WoN8r&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs22.pdf",
"url": "https://drive.google.com/uc?id=17IgyJADA65F40kH79S90QgEzPa7IERXx&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs21.pdf",
"url": "https://drive.google.com/uc?id=1kB4eFfLisPXKOirGUHbbcOyf73t7MVqL&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs20.pdf",
"url": "https://drive.google.com/uc?id=179j9tD1xRSgd9COM7rzErO6FLYO2sc_a&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs19.pdf",
"url": "https://drive.google.com/uc?id=1ogZUTfKjvQhg58GXtVUxe1U8VvHU-3ap&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs18.pdf",
"url": "https://drive.google.com/uc?id=1MfuvRhAnhMEMbxn5js2ffWqMHJx2fsd2&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/customer-information-sheet-for-inward-payments-to-hong-kong.pdf",
"url": "https://drive.google.com/uc?id=1s-H3an7HLBM9ku6d6Hcdj1qkSwKAHngU&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf15.pdf",
"url": "https://drive.google.com/uc?id=1rRQXo9XHnCVTG8XqNAv0SJwPTW36MMbm&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf16.pdf",
"url": "https://drive.google.com/uc?id=18ljRqkdyXEZ464E0dpKjaEa2NFexyw3I&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf17.pdf",
"url": "https://drive.google.com/uc?id=1VrqOnyhpOkMpyIJ6YMrAhixpahjYonOd&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf18.pdf",
"url": "https://drive.google.com/uc?id=182CLDUr372-jpAiY4YvSbGNXF9TsWxzA&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf19.pdf",
"url": "https://drive.google.com/uc?id=1YkJtjlklKN0NmLiI2Hi4f_dKtTm5SPxT&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf20.pdf",
"url": "https://drive.google.com/uc?id=1a7Uc7VCMlEX6fy-5oqE6i1YLitBe7gaf&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf21.pdf",
"url": "https://drive.google.com/uc?id=1s8km4Wle4lc5PkbUQfivBFK0IJQgxMiB&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf22.pdf",
"url": "https://drive.google.com/uc?id=1HTEb1uK7LNvbVyeXgO8WemCPPgiASKiy&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf23.pdf",
"url": "https://drive.google.com/uc?id=1XM-jZlfu_i4waDZHb8Z6Vr5b3LgULTtP&export=download"
},
{
"path": "/home/user/Documents/Fundings/supported_rate.xlsx",
"url": "https://drive.google.com/uc?id=1jkC80XAlm4V_zZF4m75MGDqnVi9Pq-Ft&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/Fundings/supported_rate.xlsx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 5
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/Fundings/grf"
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/Fundings/ecs"
}
}
],
"trajectory": "trajectories/881deb30-9549-4583-a841-8270c65f2a17",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Documents/Fundings",
"/home/user/Documents/Fundings/supported_rate.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Documents/Fundings/supported_rate.xlsx",
"/home/user/Documents/Fundings/supported_rate-Sheet1.csv"
],
"dest": [
"supported_rate.xlsx",
"supported_rate-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=11mBEe1TnJhtHD5QolFuQhPvRCOSpm8ko&export=download",
"https://drive.google.com/uc?id=1len3Jz4cAztVC0t9dUyECCGecRrCsIHj&export=download"
],
"dest": [
"supported_rate-gt.xlsx",
"supported_rate-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}

3
evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json
View File

@@ -49,7 +49,8 @@
],
"trajectory": "trajectories/9219480b-3aed-47fc-8bac-d2cffc5849f7",
"related_apps": [
"vs_code",
"os"
],
"evaluator": {
"postconfig": [],

47
evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json
View File

@@ -1,7 +1,7 @@
{
"id": "d1acdb87-bb67-4f30-84aa-990e56a09c92",
"snapshot": "libreoffice_calc",
"instruction": "Can you help me find these restaurants on Google Maps? I have a list of places I've been wanting to visit, and I need to gather some detailed information about them, including the address, what people have to say about them (ratings), and the hours of operation. It would be great if you could search for each of them on Google Maps and help me jot down this information into my sheet file, thanks!",
"instruction": "Hello! I'm eagerly planning a culinary adventure to Hong Kong and have curated a list of must-visit restaurants that I've been longing to explore. However, I could use some assistance in compiling a few essential details about these establishments. Would you be so kind as to help me out? It would be fantastic if you could search for these restaurants on Google Maps. I'm particularly interested in obtaining their addresses, any available websites, and contact phone numbers. If you could gather this information and input it into my form file, I would be immensely grateful. Many thanks in advance!",
"source": "authors",
"config": [
{
@@ -52,13 +52,54 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [],
"func": "",
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Desktop",
"/home/user/Desktop/MUST_VISIT.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/MUST_VISIT.xlsx",
"/home/user/Desktop/MUST_VISIT-Sheet1.csv"
],
"dest": [
"MUST_VISIT.xlsx",
"MUST_VISIT-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1MV6jBvRbbYwPqeFTd_nX40xzyltNhphl&export=download",
"https://drive.google.com/uc?id=1CGoRQDLw9-Ai7daq3qCz0o9kYSZB2WNn&export=download"
],
"dest": [
"MUST_VISIT-gt.xlsx",
"MUST_VISIT-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}