lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source 
# Conflicts:
#	desktop_env/controllers/setup.py
This commit is contained in:
Timothyxxx
2024-01-15 13:51:11 +08:00
parent 24169a65d0 a9e9ebbc80
commit 1141232d80
12 changed files with 356 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
desktop_env/controllers/python.py
View File

@@ -197,8 +197,10 @@ class PythonController:
if "text" not in parameters:
raise Exception(f"Unknown parameters: {parameters}")
# deal with special ' and \ characters
text = parameters["text"].replace("\\", "\\\\").replace("'", "\\'")
self.execute_python_command(f"pyautogui.typewrite('{text}')")
# text = parameters["text"].replace("\\", "\\\\").replace("'", "\\'")
# self.execute_python_command(f"pyautogui.typewrite('{text}')")
text = parameters["text"]
self.execute_python_command("pyautogui.typewrite({:})".format(repr(text)))
elif action_type == "PRESS":
if "key" not in parameters:

10
desktop_env/controllers/setup.py
View File

@@ -205,11 +205,15 @@ class SetupController:
except requests.exceptions.RequestException as e:
logger.error("An error occurred while trying to send the request: %s", e)
def _launch_setup(self, command: Union[str, List[str]]):
def _launch_setup(self, command: Union[str, List[str]], shell: bool = False):
if not command:
raise Exception("Empty command to launch.")
payload = json.dumps({"command": command})
if not shell and isinstance(command, str) and len(command.split()) > 1:
logger.warning("Command should be a list of strings. Now it is a string. Will split it by space.")
command = command.split()
payload = json.dumps({"command": command, "shell": shell})
headers = {"Content-Type": "application/json"}
try:
@@ -241,7 +245,7 @@ class SetupController:
while not terminates:
try:
response = requests.post(self.http_server_setup_root + "/execute", headers=headers, data=payload)
response = requests.post(self.http_server + "/setup" + "/execute", headers=headers, data=payload)
if response.status_code == 200:
results: Dict[str, str] = response.json()
if stdout:

1
desktop_env/evaluators/metrics/__init__.py
View File

@@ -14,3 +14,4 @@ from .gimp import increase_saturation, decrease_brightness, check_file_exists, c
from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed
from .impress import check_slide_numbers_color, compare_pptx_files, check_for_two_lines

71
desktop_env/evaluators/metrics/impress.py
View File

@@ -1,4 +1,75 @@
from pptx import Presentation
import os
def is_red_color(color):
#judge if the color is red
print(color.rgb)
return color and color.rgb == (255, 0, 0)
def get_master_placeholder_color(prs):
# get the color of the placeholder
masters = prs.slide_masters
for idx, master in enumerate(masters):
for placeholder in master.placeholders:
if placeholder.has_text_frame and placeholder.text == "<number>":
text_frame = placeholder.text_frame
if text_frame.paragraphs:
first_paragraph = text_frame.paragraphs[0]
return first_paragraph.font.color
return None
def check_slide_numbers_color(pptx_file_path):
presentation = Presentation(pptx_file_path)
for i, slide in enumerate(presentation.slides):
for shape in slide.shapes:
# check if the shape is a text box
if hasattr(shape, "text"):
if shape.text.isdigit():
# "SlidePlaceholder" is the name of the placeholder in the master slide
page_number_text = shape.text
font_color = get_master_placeholder_color(presentation)
print(font_color)
return 1 if font_color is not None and is_red_color(font_color) else 0
def compare_pptx_files(file1_path, file2_path):
prs1 = Presentation(file1_path)
prs2 = Presentation(file2_path)
# compare the number of slides
if len(prs1.slides) != len(prs2.slides):
return 0
# compare the content of each slide
for slide1, slide2 in zip(prs1.slides, prs2.slides):
# check if the shapes are the same
for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
if hasattr(shape1, "text") and hasattr(shape2, "text"):
if shape1.text != shape2.text:
return 0
return 1
def has_two_lines_on_page(slide):
line_count = 0
for shape in slide.shapes:
if shape.shape_type == 1: # 1 表示 Line 形状
line_count += 1
if line_count >= 2:
return True
return False
def check_for_two_lines(prs):
prs = Presentation(prs)
for i, slide in enumerate(prs.slides):
if has_two_lines_on_page(slide):
return 1
return 0
def check_file_exists(directory, filename):
file_path = os.path.join(directory, filename)
return 1 if os.path.isfile(file_path) else 0
if __name__ == "__main__":
path1 = "../../任务数据/LibreOffice Impress/Change_Color_Slide_Number_gold_textbox.pptx"

7
desktop_env/server/main.py
View File

@@ -63,14 +63,15 @@ def execute_command():
@app.route('/setup/launch', methods=["POST"])
def launch_app():
data = request.json
command: List[str] = data.get("command", [])
shell = data.get("shell", False)
command: List[str] = data.get("command", "" if shell else [])
if isinstance(command, str):
command = shlex.split(command)
try:
subprocess.Popen(command)
return "{:} launched successfully".format(" ".join(command))
subprocess.Popen(command, shell=shell)
return "{:} launched successfully".format(command if shell else " ".join(command))
except Exception as e:
return jsonify({"status": "error", "message": str(e)}), 500

1
desktop_env/server/requirements.txt
View File

@@ -5,3 +5,4 @@ git+https://github.com/moses-palmer/pynput.git@refs/pull/541/head # to make sure
requests
flask
numpy
lxml

32
evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json
View File

@@ -1,12 +1,34 @@
{
"id": "455d3c66-7dc6-4537-a39a-36d3e9119df7",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me export impress file to image jpg file?",
"instruction": "Could you help me export an Impress file to a .jpg image file and save it as res.jpg on the Desktop? ",
"source": "https://stackoverflow.com/questions/75626383/how-export-libreoffice-impress-to-image",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=12MxMjw28_t1nTLihlDpToKebjsSDsjwx&export=download&authuser=0&confirm=t&uuid=1ccc1da0-d7c7-494f-a0e3-59eb55f54e3f&at=APZUnTXvNIRMsF2cjZuFxmQzByhC:1705253210291",
"path": "Desktop/wssf-project-plan-on-a-page.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/wssf-project-plan-on-a-page.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
}
"evaluator": {
"func": "check_file_exists",
"file_name": "res.png",
"directory": "/home/user/Desktop/"
}
}

35
evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
View File

@@ -1,12 +1,37 @@
{
"id": "550ce7e7-747b-495f-b122-acdc4d0b8e54",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me add a strike-through on this text",
"instruction": "I am checking our soccer club's to-do list for the last semester and adding strike-through sign on the line we have already accomplished. Could you help me add a strike-through on the first and second line?",
"source": "https://superuser.com/questions/1211035/libreoffice-impress-animations-how-to-strikethrough-on-click?rq=1",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fw0baVZ15s0r1WGEBftgED2H0ljZgYtu&export=download&authuser=0&confirm=t&uuid=df03788a-81ef-4e55-b33a-2fba7ab28cb8&at=APZUnTXPb-sm88KNwmNeugbhPrzx:17052529805399",
"path": "Desktop/New_Club_Spring_2018_Training.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/New_Club_Spring_2018_Training.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
}
"evaluator": {
"func": "check_for_two_lines",
"result": {
"type": "vm_file",
"path": "Desktop/New_Club_Spring_2018_Training.pptx",
"dest": "New_Club_Spring_2018_Training.pptx"
}
}
}

40
evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json
View File

@@ -1,12 +1,42 @@
{
"id": "5d901039-a89c-4bfb-967b-bf66f4df075e",
"snapshot": "libreoffice_impress",
"instruction": "Help me stretch the image to fill the entire page, keeping its proportion and centering the image",
"instruction": "I want to make this page my cover page. Could you help me stretch this image to fill the entire page, keeping its proportion and centering the image.",
"source": "https://superuser.com/questions/986776/how-can-i-stretch-an-image-in-a-libreoffice-impress-presentation-to-fill-the-pag",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=16K6TpGIRZpqOJUu-mtJQ_78kIwLcn-4D&export=download&authuser=0&confirm=t&uuid=945b6f33-53d2-4e87-ada9-efa8b938a499&at=APZUnTVw4fKyJPW0vAAJURruAJIP:1705250184439",
"path": "Desktop/CPD_Background_Investigation_Process.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/CPD_Background_Investigation_Process.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
}
"evaluator": {
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1rsvFPyHYiIPh1c8Nj8say0NJCG2VIDr7&export=download&authuser=0&confirm=t&uuid=aac08a92-6595-47d8-84dc-8f1ab1df987f&at=APZUnTXIWCn5B0CpLttvG2bsr_a7:1705250423565",
"dest": "CPD_Background_Investigation_Process_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/CPD_Background_Investigation_Process.pptx",
"dest": "CPD_Background_Investigation_Process.pptx"
}
}
}

2
evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json
View File

@@ -1,7 +1,7 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "thunderbird",
"instruction": "Could you help me back up all the email files in my profile to ~/email.bak? Please save them separately in eml format.",
"instruction": "Could you help me back up all the email files in my profile to ~/emails.bak? Please save them separately in eml format.",
"source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird",
"config": [
{

98
evaluation_examples/examples/thunderbird/d088f539-cab4-4f9a-ac92-9999fc3a656e.json Normal file
View File

@@ -0,0 +1,98 @@
{
"id": "d088f539-cab4-4f9a-ac92-9999fc3a656e",
"snapshot": "thunderbird",
"instruction": "Attach the my AWS bill to the email. The bill is stored at ~/aws-bill.pdf. Don't close it or send it. I haven't finish all the contents.",
"source": "https://support.mozilla.org/en-US/kb/how-use-attachments",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1JDUxe8Cs0D9La9ECoYbHJAIQ95DZfkDT&export=download&authuser=0&confirm=t&uuid=1e0f94d8-4fb1-4f9b-878a-516cf4a7ebc4&at=APZUnTXHBZKTuW_vQUzi-1XKuAYg:1705242554801",
"path": "/home/user/aws-bill.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
},
{
"url": "https://drive.usercontent.google.com/download?id=17LutYTkEXhM__sxvrdldar7p-eQba4Pm&export=download&authuser=0&confirm=t&uuid=34730e1c-0fac-401a-ab91-102a47f02224&at=APZUnTX4eFTnG9vnYlQEJX6N0jqP:1705227498314",
"path": "/home/user/.aws-bill-mail-body.html"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',to=assistant@outlook.com,subject='New-month AWS Bill',body='$(cat /home/user/.aws-bill-mail-body.html)'\"",
"shell": true
}
}
],
"trajectory": "trajectories/d088f539-cab4-4f9a-ac92-9999fc3a656e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1abb7PPPRGGy3elpq2vTXe2aKTMOrbm7c&export=download&authuser=0&confirm=t&uuid=652a3d4c-278e-499c-bb26-58d0f70aa317&at=APZUnTXs_ZTZzOF3QrBa_rMjMr08:1705287009269",
"path": "/home/user/show-thunderbird-attachments.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["pip", "install", "cssselect"]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"/home/user/show-thunderbird-attachments.py",
"New-month AWS Bill",
"aws-bill.pdf"
],
"stdout": "check-attachments.output"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "check-attachments.output"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"Attachment added!"
]
}
}
}
}

77
evaluation_examples/examples/thunderbird/d088f539-cab4-4f9a-ac92-9999fc3a656e.json.nosetup Normal file
View File

@@ -0,0 +1,77 @@
{
"id": "d088f539-cab4-4f9a-ac92-9999fc3a656e",
"snapshot": "thunderbird",
"instruction": "Attach the my AWS bill to the email. The bill is stored at ~/aws-bill.pdf. Don't close it or send it. I haven't finish all the contents.",
"source": "https://support.mozilla.org/en-US/kb/how-use-attachments",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1JDUxe8Cs0D9La9ECoYbHJAIQ95DZfkDT&export=download&authuser=0&confirm=t&uuid=1e0f94d8-4fb1-4f9b-878a-516cf4a7ebc4&at=APZUnTXHBZKTuW_vQUzi-1XKuAYg:1705242554801",
"path": "/home/user/aws-bill.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
},
{
"url": "https://drive.usercontent.google.com/download?id=17LutYTkEXhM__sxvrdldar7p-eQba4Pm&export=download&authuser=0&confirm=t&uuid=34730e1c-0fac-401a-ab91-102a47f02224&at=APZUnTX4eFTnG9vnYlQEJX6N0jqP:1705227498314",
"path": "/home/user/.aws-bill-mail-body.html"
}
]
}
}
],
"trajectory": "trajectories/d088f539-cab4-4f9a-ac92-9999fc3a656e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1abb7PPPRGGy3elpq2vTXe2aKTMOrbm7c&export=download&authuser=0&confirm=t&uuid=652a3d4c-278e-499c-bb26-58d0f70aa317&at=APZUnTXs_ZTZzOF3QrBa_rMjMr08:1705287009269",
"path": "/home/user/show-thunderbird-attachments.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["pip", "install", "cssselect"]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"/home/user/show-thunderbird-attachments.py",
"New-month AWS Bill",
"aws-bill.pdf"
],
"stdout": "check-attachments.output"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "check-attachments.output"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"Attachment added!"
]
}
}
}
}