lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan17thv2

Browse Source 
updated compare_table with compare the shown value through exported csv
This commit is contained in:
David Chang
2024-01-17 22:43:26 +08:00
parent ffc4c32bac
commit 19214f2107
7 changed files with 371 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
desktop_env/evaluators/getters/file.py
View File

@@ -1,50 +1,92 @@
import os
from typing import Dict
from typing import Optional
from typing import Dict, List, Set
from typing import Optional, Any, Union
import requests
def get_cloud_file(env, config: Dict[str, str]) -> str:
def get_cloud_file(env, config: Dict[str, Any]) -> Union[str, List[str]]:
"""
Config:
path (str): the url to download from
dest (str): file name of the downloaded file
path (str|List[str]): the url to download from
dest (str|List[str])): file name of the downloaded file
multi (bool) : optional. if path and dest are lists providing
information of multiple files. defaults to False
gives (List[int]): optional. defaults to [0]. which files are directly
returned to the metric. if len==1, str is returned; else, list is
returned.
"""
_path = os.path.join(env.cache_dir, config["dest"])
if os.path.exists(_path):
return _path
if not config.get("multi", False):
paths: List[str] = [config["path"]]
dests: List[str] = [config["dest"]]
else:
paths: List[str] = config["path"]
dests: List[str] = config["dest"]
cache_paths: List[str] = []
url = config["path"]
response = requests.get(url, stream=True)
response.raise_for_status()
gives: Set[int] = set(config.get("gives", [0]))
with open(_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
for i, (p, d) in enumerate(zip(paths, dests)):
_path = os.path.join(env.cache_dir, d)
if i in gives:
cache_paths.append(_path)
return _path
if os.path.exists(_path):
#return _path
continue
url = p
response = requests.get(url, stream=True)
response.raise_for_status()
with open(_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
return cache_paths[0] if len(cache_paths)==1 else cache_paths
def get_vm_file(env, config: Dict[str, str]) -> Optional[str]:
def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Optional[str]]]:
"""
Config:
path (str): absolute path on the VM to fetch
dest (str): file name of the downloaded file
multi (bool) : optional. if path and dest are lists providing
information of multiple files. defaults to False
gives (List[int]): optional. defaults to [0]. which files are directly
returned to the metric. if len==1, str is returned; else, list is
returned.
"""
_path = os.path.join(env.cache_dir, config["dest"])
if not config.get("multi", False):
paths: List[str] = [config["path"]]
dests: List[str] = [config["dest"]]
else:
paths: List[str] = config["path"]
dests: List[str] = config["dest"]
cache_paths: List[str] = []
file = env.controller.get_file(config["path"])
if file is None:
return None
# raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
with open(_path, "wb") as f:
f.write(file)
gives: Set[int] = set(config.get("gives", [0]))
return _path
for i, (p, d) in enumerate(zip(paths, dests)):
_path = os.path.join(env.cache_dir, d)
file = env.controller.get_file(p)
if file is None:
#return None
# raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
if i in gives:
cache_paths.append(None)
continue
if i in gives:
cache_paths.append(_path)
with open(_path, "wb") as f:
f.write(file)
return cache_paths[0] if len(cache_paths)==1 else cache_paths
def get_cache_file(env, config: Dict[str, str]) -> str:

44
desktop_env/evaluators/metrics/table.py
View File

@@ -3,6 +3,8 @@ import operator
from numbers import Number
from typing import Any, Union
from typing import Dict, List
import os.path
import itertools
import openpyxl
import pandas as pd
@@ -26,6 +28,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
* chart
* number_format
"chart_props": list of str, giving the converned chart properties
"as_shown": bool, TODO
}
Return:
@@ -35,10 +38,35 @@ def compare_table(actual: str, expected: str, **options) -> float:
if actual is None:
return 0.
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
logger.debug("Normal Contents Metric: {:}".format(metric))
if options.get("as_shown", False):
expected_csv: str = os.path.splitext(expected)[0] + ".csv"
actual_csv: str = os.path.splitext(actual)[0] + ".csv"
with open(expected_csv) as f:
expected_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0
, map( lambda l: l.strip()
, reversed(f.read().splitlines())
)
)
)
if options.get("ignore_case", False):
expected_lines = [l.lower() for l in expected_lines]
with open(actual_csv) as f:
actual_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0
, map( lambda l: l.strip()
, reversed(f.read().splitlines())
)
)
)
if options.get("ignore_case", False):
actual_lines = [l.lower() for l in actual_lines]
metric: bool = expected_lines==actual_lines
logger.debug("Content Metric just as shown: %s", metric)
else:
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
logger.debug("Normal Content Metric: {:}".format(metric))
features: List[str] = options.get("features", [])
for ftr in features:
@@ -219,6 +247,8 @@ if __name__ == '__main__':
# print(check_zoom(path1, {"relation": "lt", "ref_value": 100}))
# print(check_zoom(path2, {"relation": "lt", "ref_value": 100}))
path1 = "../../任务数据/LibreOffice Calc/Padding_Decimals_In_Formular_gold.xlsx"
data_frame: pd.DataFrame = pd.read_excel(path1)
print(data_frame)
path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx"
path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx"
#data_frame: pd.DataFrame = pd.read_excel(path1)
#print(data_frame)
print(compare_table(path1, path2, as_shown=True))

103
evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
View File

@@ -3,20 +3,101 @@
"snapshot": "libreoffice_calc",
"instruction": "I would like to pad all the numbers in the 'Old ID' column with zeros in front, to fill them up to seven digits in the 'New 7 Digit ID' column.",
"source": "https://www.youtube.com/shorts/FPAQaDTS8VY",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx"
]
"config": [
{
"type": "download",
"parameters": {
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=1DqGy5JRKOuZMRJ8O76d4Cds4WaRyz8V1&export=download&authuser=0&confirm=t&uuid=fa0694d1-2a77-4fd2-89d3-d9b854317823&at=APZUnTU9BxqG7E8tLZ104c0E8BEL:1705501029016",
"path": "/home/user/Customers_New_7digit_Id.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Customers_New_7digit_Id.xlsx"
}
}
},
"trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Customers_New_7digit_Id.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Customers_New_7digit_Id.xlsx",
"/home/user/Customers_New_7digit_Id.csv"
],
"dest": [
"Customers_New_7digit_Id.xlsx",
"Customers_New_7digit_Id.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=1zXz5k5A403IR0GE6DRRXRgQZrSIoVFSz&export=download&authuser=0&confirm=t&uuid=ba70b841-969c-4d91-9288-0011aeecf251&at=APZUnTWx3LL9udCgJAh-VMFzzfod:1705501007861",
"https://drive.usercontent.google.com/download?id=1h1GnUpyj92K7FXiHJ1xVaUYW_UYMDLPM&export=download&authuser=0&confirm=t&uuid=98de75bd-ba32-4ceb-97a7-b8d303a8dc96&at=APZUnTUewdfFIsyC3UlSlMcmmRbo:1705500978790"
],
"dest": [
"Customers_New_7digit_Id_gold.xlsx",
"Customers_New_7digit_Id_gold.csv"
],
"multi": true
},
"options": {
"as_shown": true
}
}
}

46
evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
View File

@@ -50,23 +50,55 @@
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Represent_in_millions_billions.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/Represent_in_millions_billions.xlsx",
"dest": "Represent_in_millions_billions.xlsx"
"path": [
"/home/user/Represent_in_millions_billions.xlsx",
"/home/user/Represent_in_millions_billions.csv"
],
"dest": [
"Represent_in_millions_billions.xlsx",
"Represent_in_millions_billions.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995",
"dest": "Represent_in_millions_billions_gold.xlsx"
"path": [
"https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995",
"https://drive.usercontent.google.com/download?id=1CjzmJ4vqwPI7DMTR_6a8ytO-W-4xavBS&export=download&authuser=0&confirm=t&uuid=32c3430a-6d3d-47cc-bb5c-e683963b19f7&at=APZUnTWSBLNtkrZTix_BRl9gyaek:1705497493679"
],
"dest": [
"Represent_in_millions_billions_gold.xlsx",
"Represent_in_millions_billions_gold.csv"
],
"multi": true
},
"options": {
"features": [
"number_format"
]
"as_shown": true,
"ignore_case": true
}
}
}

64
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
View File

@@ -1,7 +1,7 @@
{
"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
"snapshot": "libreoffice_calc",
"instruction": "Could you help me setting decimal separator as a dot (.)?",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
"config": [
{
@@ -14,6 +14,12 @@
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Set_Decimal_Separator_Dot.xlsx"
}
}
],
"trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25",
@@ -44,26 +50,54 @@
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Set_Decimal_Separator_Dot.xlsx"
]
}
}
],
"func": "check_libre_locale",
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/.config/libreoffice/4/user/registrymodifications.xcu",
"dest": "registrymodifications.xcu"
"path": [
"/home/user/Set_Decimal_Separator_Dot.xlsx",
"/home/user/Set_Decimal_Separator_Dot.csv"
],
"dest": [
"Set_Decimal_Separator_Dot.xlsx",
"Set_Decimal_Separator_Dot.csv"
],
"multi": true
},
"expected": {
"type": "rule",
"rules": {
"locale_set": [
"ru-*",
"de-*",
"fr-*",
"pt-*",
"es-*",
"it-*"
]
}
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975",
"https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507"
],
"dest": [
"Set_Decimal_Separator_Dot_gold.xlsx",
"Set_Decimal_Separator_Dot_gold.csv"
],
"multi": true
},
"options": {
"as_shown": true
}
}
}

85
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup Normal file
View File

@@ -0,0 +1,85 @@
{
"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
"snapshot": "libreoffice_calc",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
"config": [],
"trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25",
"related_apps": [
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Set_Decimal_Separator_Dot.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Set_Decimal_Separator_Dot.xlsx",
"/home/user/Set_Decimal_Separator_Dot.csv"
],
"dest": [
"Set_Decimal_Separator_Dot.xlsx",
"Set_Decimal_Separator_Dot.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975",
"https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507"
],
"dest": [
"Set_Decimal_Separator_Dot_gold.xlsx",
"Set_Decimal_Separator_Dot_gold.csv"
],
"multi": true
},
"options": {
"as_shown": true
}
}
}

4
main.py
View File

@@ -46,9 +46,9 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/thunderbird/d088f539-cab4-4f9a-ac92-9999fc3a656e.json.nosetup", "r") as f:
with open("evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup", "r") as f:
example = json.load(f)
example["snapshot"] = "Snapshot 24"
example["snapshot"] = "Snapshot 26"
env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
, action_space="computer_13"