lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-03-01 15:11:41 +08:00
parent f0a20cdbd1 58e21ab871
commit 96188217c1
6 changed files with 227 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/evaluators/metrics/__init__.py
View File

@@ -57,7 +57,8 @@ from .general import (
is_in_list,
fuzzy_match,
check_include_exclude,
check_direct_json_object
check_direct_json_object,
diff_text_file
)
from .gimp import (
check_brightness_decrease_and_structure_sim,

24
desktop_env/evaluators/metrics/general.py
View File

@@ -12,6 +12,7 @@ import lxml.etree
from lxml.cssselect import CSSSelector
from lxml.etree import _Element
from rapidfuzz import fuzz
import difflib
from .utils import _match_record, _match_value_to_rule
@@ -45,7 +46,15 @@ def is_in_list(result, rules) -> float:
else:
return 0.
def diff_text_file(result: str, expect: str) -> float:
if result is None:
return 0.
with open(result) as f:
result_lines: List[str] = f.read().splitlines()
with open(expect) as f:
expected_lines: List[str] = f.read().splitlines()
return difflib.SequenceMatcher(a=result_lines, b=expected_lines).ratio()
def fuzzy_match(result, rules) -> float:
expect = rules["expected"]
@@ -212,14 +221,21 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
for r in expect_rules:
value = result
for k in r["key"]:
value = value[k]
try:
value = value[k]
except KeyError:
return 0.
metric = metric and _match_value_to_rule(value, r)
for r in unexpect_rules:
value = result
for k in r["key"]:
value = value[k]
try:
value = value[k]
except KeyError:
value = None
break
metric = metric and not _match_value_to_rule(value, r)
return metric
return float(metric)
def check_direct_json_object(result, rules)->float:
@@ -238,4 +254,4 @@ def check_direct_json_object(result, rules)->float:
expected_value = expected_json.get(key)
if expected_value != result.get(key):
return 0.
return 1.0
return 1.0

1
desktop_env/evaluators/metrics/table.py
View File

@@ -122,7 +122,6 @@ def compare_table(result: str, expected: str = None, **options) -> float:
worksheetr_names: List[str] = pdworkbookr.sheet_names
if expected is not None:
xlworkbooke: Workbook = openpyxl.load_workbook(filename=expected)
pdworkbooke = pd.ExcelFile(expected)
worksheete_names: List[str] = pdworkbooke.sheet_names

4
desktop_env/evaluators/metrics/utils.py
View File

@@ -591,7 +591,7 @@ def _match_value_to_rule(value: V, rule: Dict[str, Union[str, V]]) -> bool:
bool
"""
if rule["method"].startswith("re"):
if rule["method"].startswith("re"): # re.FLAGs
flags: List[str] = rule["method"].split(".")[1:]
flags: Iterable[re.RegexFlag] = (getattr(re, fl) for fl in flags)
flag: re.RegexFlag = functools.reduce(operator.or_, flags, re.RegexFlag(0))
@@ -604,7 +604,7 @@ def _match_value_to_rule(value: V, rule: Dict[str, Union[str, V]]) -> bool:
, "ge", "gt"
}:
return getattr(operator, rule["method"])(value, rule["ref"])
if rule["method"].startswith("approx"):
if rule["method"].startswith("approx"): # approx:THRESHOLD
threshold: float = float(rule["method"].split(":")[1])
logger.debug("Approx: TH%f, REF%f, VAL%s", threshold, rule["ref"], repr(value))
try:

107
evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json Normal file
View File

@@ -0,0 +1,107 @@
{
"id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"snapshot": "chrome",
"instruction": "Help me to get an initial setup of web extension with help of https://webext.eu . Tag the extension as \"happy-extension v0.0.1\". Leave description as blank for now. A background script and browser action is needed, while other features are not. Place the auto-generated folder under folder \"~/Projects\".",
"source": "authors",
"config": [],
"trajectory": "trajectories/74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"related_apps": [
"chrome",
"os"
],
"evaluator": {
"func": [
"check_json",
"diff_text_file",
"diff_text_file",
"diff_text_file",
"diff_text_file"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/manifest.json",
"dest": "manifest.json"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/background_script.js",
"dest": "background_script.js"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/index.html",
"dest": "index.html"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/style.css",
"dest": "style.css"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/script.js",
"dest": "script.js"
}
],
"expected": [
{
"type": "rule",
"rules": {
"expect": [
{
"key": ["name"],
"method": "eq",
"ref": "happy-extension"
},
{
"key": ["version"],
"method": "eq",
"ref": "0.0.1"
},
{
"key": ["background", "scripts"],
"method": "eq",
"ref": ["background_script.js"]
},
{
"key": ["browser_action", "default_icon"],
"method": "eq",
"ref": {"64": "icons/icon.png"}
},
{
"key": ["browser_action", "default_popup"],
"method": "eq",
"ref": "browserAction/index.html"
},
{
"key": ["browser_action", "default_title"],
"method": "eq",
"ref": "happy-extension"
}
]
}
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1t5Llhn6seDUXVs-eILu6CjwFEQL9Z5Qm&export=download",
"dest": "background_script.js"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=19fMAsWd6q4ElLdOceJ-otHbxRJA_pc_U&export=download",
"dest": "index.html"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1fwfiRPjdug8uh6z23RFO1JtlGH_L_Hl_&export=download",
"dest": "style.css"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=14YYnhCfRtHQNk8M4fBPaUQeteoFMGBsA&export=download",
"dest": "script.js"
}
]
}
}

96
evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json Normal file
View File

@@ -0,0 +1,96 @@
{
"id": "7e287123-70ca-47b9-8521-47db09b69b14",
"snapshot": "libreoffice_calc",
"instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for the past five years in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Fundings/ecs", "/home/user/Documents/Fundings/grf"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Fundings/ecs/ecs15.pdf", "url": "https://drive.google.com/uc?id=1FTiT3mLlkehe2yWVdSMWr1w4ltLtQZUy&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs16.pdf", "url": "https://drive.google.com/uc?id=1DMzZyhDey3lDsQ7fcPiJm9AYGECZms3q&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs17.pdf", "url": "https://drive.google.com/uc?id=1TgAWk7FiV8fNrG2L3-Eu7BDccqtXebXY&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs23.pdf", "url": "https://drive.google.com/uc?id=11DVxH4eRjECUxZNbUhYNJhgD0Y5WoN8r&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs22.pdf", "url": "https://drive.google.com/uc?id=17IgyJADA65F40kH79S90QgEzPa7IERXx&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs21.pdf", "url": "https://drive.google.com/uc?id=1kB4eFfLisPXKOirGUHbbcOyf73t7MVqL&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs20.pdf", "url": "https://drive.google.com/uc?id=179j9tD1xRSgd9COM7rzErO6FLYO2sc_a&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs19.pdf", "url": "https://drive.google.com/uc?id=1ogZUTfKjvQhg58GXtVUxe1U8VvHU-3ap&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs18.pdf", "url": "https://drive.google.com/uc?id=1MfuvRhAnhMEMbxn5js2ffWqMHJx2fsd2&export=download"},
{"path": "/home/user/Documents/Fundings/grf/customer-information-sheet-for-inward-payments-to-hong-kong.pdf", "url": "https://drive.google.com/uc?id=1s-H3an7HLBM9ku6d6Hcdj1qkSwKAHngU&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf15.pdf", "url": "https://drive.google.com/uc?id=1rRQXo9XHnCVTG8XqNAv0SJwPTW36MMbm&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf16.pdf", "url": "https://drive.google.com/uc?id=18ljRqkdyXEZ464E0dpKjaEa2NFexyw3I&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf17.pdf", "url": "https://drive.google.com/uc?id=1VrqOnyhpOkMpyIJ6YMrAhixpahjYonOd&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf18.pdf", "url": "https://drive.google.com/uc?id=182CLDUr372-jpAiY4YvSbGNXF9TsWxzA&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf19.pdf", "url": "https://drive.google.com/uc?id=1YkJtjlklKN0NmLiI2Hi4f_dKtTm5SPxT&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf20.pdf", "url": "https://drive.google.com/uc?id=1a7Uc7VCMlEX6fy-5oqE6i1YLitBe7gaf&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf21.pdf", "url": "https://drive.google.com/uc?id=1s8km4Wle4lc5PkbUQfivBFK0IJQgxMiB&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf22.pdf", "url": "https://drive.google.com/uc?id=1HTEb1uK7LNvbVyeXgO8WemCPPgiASKiy&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf23.pdf", "url": "https://drive.google.com/uc?id=1XM-jZlfu_i4waDZHb8Z6Vr5b3LgULTtP&export=download"}
]
}
}
],
"trajectory": "trajectories/7e287123-70ca-47b9-8521-47db09b69b14",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Desktop",
"/home/user/Desktop/GRF-p5y.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/GRF-p5y.xlsx",
"/home/user/Desktop/GRF-p5y-Sheet1.csv"
],
"dest": [
"GRF-p5y.xlsx",
"GRF-p5y-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1fDM4Y-WuFCnfksPLgynj-WSmzbqn2TcV&export=download",
"https://drive.google.com/uc?id=1waThupubGOJop0FU0b0yhT6QnjNYkLGy&export=download"
],
"dest": [
"GRF-p5y-gt.xlsx",
"GRF-p5y-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}