lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2024-01-27 19:49:41 +08:00
parent 773c5ed40c 8025bf19f0
commit 50c346af11
83 changed files with 896 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/controllers/python.py
View File

@@ -11,7 +11,7 @@ logger = logging.getLogger("desktopenv.pycontroller")
class PythonController:
def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; {command}"):
def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; import time; {command}"):
self.vm_ip = vm_ip
self.http_server = f"http://{vm_ip}:5000"
self.pkgs_prefix = pkgs_prefix # fixme: this is a hacky way to execute python commands. fix it and combine it with installation of packages

2
desktop_env/evaluators/getters/__init__.py
View File

@@ -2,7 +2,7 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g
get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \
get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line
from .general import get_vm_command_line, get_vm_terminal_output
from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree

4
desktop_env/evaluators/getters/general.py
View File

@@ -18,3 +18,7 @@ def get_vm_command_line(env, config: Dict[str, str]):
else:
logger.error("Failed to get vm command line. Status code: %d", response.status_code)
return None
def get_vm_terminal_output(env, config: Dict[str, str]):
return env.controller.get_terminal_output()

4
desktop_env/server/main.py
View File

@@ -5,7 +5,7 @@ import shlex
import subprocess
from pathlib import Path
from typing import Any, Optional
from typing import List, Dict
from typing import List, Dict, Tuple
import Xlib
import lxml.etree
@@ -16,7 +16,7 @@ from PIL import Image
from Xlib import display, X
from flask import Flask, request, jsonify, send_file, abort # , send_from_directory
from lxml.etree import _Element
from pyatspi import Accessible, StateType
from pyatspi import Accessible, StateType, STATE_SHOWING
from pyatspi import Action as ATAction
from pyatspi import Component, Document
from pyatspi import Text as ATText

2
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},

2
evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},

2
evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},

2
evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},

2
evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},

2
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json
View File

@@ -29,7 +29,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},

2
evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

2
evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}

25
evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Novels_Intro_Packet.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",

25
evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "H2O_Factsheet_WA.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_files",
"result": {
"type": "vm_file",

25
evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "LibreOffice_Open_Source_Word_Processing.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "has_page_numbers_in_footers",
"result": {
"type": "vm_file",

25
evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Dublin_Zoo_Intro.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_font_names",
"expected": {
"type": "rule",

25
evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Constitution_Template_With_Guidelines.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "is_first_line_centered",
"result": {
"type": "vm_file",

27
evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "How_to_read_a_scientific_article.docx"
"path": "Desktop/How_to_read_a_scientific_article.docx"
}
}
],
@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "How_to_read_a_scientific_article.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": [
"compare_highlighted_text",
"compare_docx_files"

25
evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "NOVEL_Submission_Guidelines.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",

14
evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
View File

@@ -27,8 +27,16 @@
"libreoffice_writer"
],
"evaluator": {
"func": "check_file_exists",
"file_name": "View_Person_Organizational_Summary.pdf",
"directory": "/home/user/Downloads/"
"func": "compare_pdfs",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Spn-Gw7D-bRvV4udNQoGNEOViUqf6bL0&export=download&authuser=0&confirm=t&uuid=dcc0eb01-89ed-4852-a7cb-d0400d977ac8&at=APZUnTX57XnHwmb-y3m4JdNkvu6z:1706328786805",
"dest": "Constitution_Template_With_Guidelines_Gold.pdf"
},
"result": {
"type": "vm_file",
"path": "Desktop/View_Person_Organizational_Summary.pdf",
"dest": "Constitution_Template_With_Guidelines.pdf"
}
}
}

25
evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Factoring_Perfect_Square_Trinomials.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_insert_equation",
"expected": {
"type": "cloud_file",

37
evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579",
"url": "https://drive.usercontent.google.com/download?id=1yeD0YIFCSEAdi6MPiiqLe6F4a19I5wZP&export=download&authuser=0&confirm=t&uuid=b6693633-00d7-4a8b-b35e-77ec452dd6a3&at=APZUnTVHRdoysZFBTmGmX6rCJLHK:1706330570587",
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
]
@@ -20,6 +20,16 @@
"parameters": {
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(1); pyautogui.press(\"down\", presses=38, interval=0.1); time.sleep(1); pyautogui.scroll(-2)"
]
}
}
],
"trajectory": "trajectories/",
@@ -27,6 +37,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Table_Of_Work_Effort_Instructions.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_tables",
"expected": {
"type": "cloud_file",

35
evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
View File

@@ -31,6 +31,16 @@
"parameters": {
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(1); pyautogui.press(\"down\", presses=8); time.sleep(1); pyautogui.scroll(-2)"
]
}
}
],
"trajectory": "trajectories/",
@@ -38,6 +48,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Viewing_Your_Class_Schedule_and_Textbooks.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_contains_image",
"result": {
"type": "vm_file",

25
evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Graphemes_Sound_Letter_Patterns_Gold.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_tables",
"expected": {
"type": "cloud_file",

27
evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
View File

@@ -1,7 +1,7 @@
{
"id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
"snapshot": "libreoffice_writer",
"instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).",
"instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference in the fourth paragraph where I marked \"<add here>\".",
"source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/",
"config": [
{
@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Essay_Writing_English_for_uni.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",

27
evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759",
"path": "Geography_And_Magical_Realism.docx"
"path": "Desktop/Geography_And_Magical_Realism.docx"
}
]
}
@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Geography_And_Magical_Realism.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",

27
evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
View File

@@ -1,7 +1,7 @@
{
"id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb",
"snapshot": "libreoffice_writer",
"instruction": "Help me insert a blank page where my cursor is located.",
"instruction": "Hey, can you throw in a blank page right after this one?",
"source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice",
"config": [
{
@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Sample_Statutory_Declaration.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "contains_page_break",
"result": {
"type": "vm_file",

67
evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json
View File

@@ -1,17 +1,56 @@
{
"id": "13584542-872b-42d8-b299-866967b5c3ef",
"snapshot": "os",
"instruction": "Set the default terminal size and screen x,y coordinates in order to make the terminal window always open at the same location and with the same size.",
"source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "13584542-872b-42d8-b299-866967b5c3ef",
"snapshot": "os",
"instruction": "I click in terminal: terminal->132x43 to change terminal size but after each reboot terminal size is set to default setting and I have to change it again. Help me set it permanently",
"source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 1
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5); pyautogui.write('stty size'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "check_include_exclude",
"result": {
"type": "vm_terminal_output"
},
"expected": {
"type": "rule",
"rules": {
"include": [
"43 132"
],
"exclude": [
]
}
}
}
}

10
evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json
View File

@@ -80,6 +80,16 @@
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [

13
evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json
View File

@@ -4,7 +4,18 @@
"instruction": "The volume of my system is too small. Can you help me turn up to the max volume?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

13
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -4,7 +4,18 @@
"instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

15
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json
View File

@@ -3,7 +3,18 @@
"snapshot": "os",
"instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
@@ -12,7 +23,7 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -d '/home/user/Desktop/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"command": "[ -d '/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {

10
evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json
View File

@@ -22,6 +22,16 @@
"command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [

13
evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json
View File

@@ -3,7 +3,18 @@
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"

10
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json
View File

@@ -17,7 +17,17 @@
"command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [

13
evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json
View File

@@ -4,7 +4,18 @@
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

10
evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json
View File

@@ -10,6 +10,16 @@
"command": "echo password | sudo -S su - charles",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

12
evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json
View File

@@ -4,6 +4,18 @@
"instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

10
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "xdg-settings set default-web-browser firefox.desktop"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json
View File

@@ -5,6 +5,16 @@
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"

13
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
View File

@@ -4,7 +4,18 @@
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

12
evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json
View File

@@ -4,6 +4,18 @@
"instruction": "I want to uninstall the Mahjongg on my system. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

10
evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json
View File

@@ -11,6 +11,16 @@
"command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [

10
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
View File

@@ -21,6 +21,16 @@
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']\""
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json
View File

@@ -10,7 +10,17 @@
"command": "gsettings set org.gnome.desktop.notifications show-banners true",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [

54
evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json
View File

@@ -1,24 +1,36 @@
{
"id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3",
"snapshot": "os",
"instruction": "I want to see the battery percentage. Can you help me display it on my screen?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "true\n"
}
"id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3",
"snapshot": "os",
"instruction": "I want to see the battery percentage. Can you help me display it on my screen?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"expected": "true\n"
}
}
}
}

10
evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json
View File

@@ -64,6 +64,16 @@
"import pyautogui; pyautogui.press('f11');"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
View File

@@ -20,6 +20,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

27
evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json
View File

@@ -4,10 +4,35 @@
"instruction": "Can you make the video fill up the whole screen? It's a bit too small to see right now, and I'd like to see it better.",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=13OhDRuINzQ-w0qmzdGJ8ZSWnKyKnTRje&export=download&authuser=0&confirm=t&uuid=39cf0297-23eb-47df-a642-fd6c2f981eff&at=APZUnTUR83eI1tenMxVdyFxNeBo7:1706275408621",
"path": "Desktop/Gen 2.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": [
"vlc",
"--start-time=15",
"Desktop/Gen 2.mp4"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],

10
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -24,6 +24,16 @@
"Desktop/Baby Justin Bieber.mp4"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json
View File

@@ -20,6 +20,16 @@
"parameters": {
"command": ["vlc", "Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4"]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
View File

@@ -26,6 +26,16 @@
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

22
evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
View File

@@ -26,6 +26,28 @@
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Interstellar Movie - Interstellar Movie - Official Trailer - VLC media player"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(500, 500); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

2
mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
View File

@@ -40,6 +40,8 @@ def filter_nodes(nodes):
filtered_nodes.append(node)
elif node.tag == 'text':
continue
elif node.get("name") == "" and node.text is None:
continue
else:
coords = tuple(
map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord').strip('()').split(', ')))

19
mm_agents/gpt_4v_agent.py
View File

@@ -39,7 +39,10 @@ def linearize_accessibility_tree(accessibility_tree):
for node in filtered_nodes:
linearized_accessibility_tree += node.tag + "\t"
linearized_accessibility_tree += node.attrib.get('name') + "\t"
linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t"
if node.text:
linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t"
else:
linearized_accessibility_tree += '""\t'
linearized_accessibility_tree += node.attrib.get(
'{uri:deskat:component.at-spi.gnome.org}screencoord') + "\t"
linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size') + "\n"
@@ -87,7 +90,7 @@ def parse_actions_from_string(input_string):
try:
action_dict = json.loads(input_string)
return [action_dict]
except json.JSONDecodeError as e:
except json.JSONDecodeError:
raise ValueError("Invalid response format: " + input_string)
@@ -226,8 +229,8 @@ class GPT4v_Agent:
})
# Append trajectory
assert len(self.observations) == len(self.actions) and len(self.actions) == len(self.thoughts)\
, "The number of observations and actions should be the same."
assert len(self.observations) == len(self.actions) and len(self.actions) == len(self.thoughts) \
, "The number of observations and actions should be the same."
if len(self.observations) > self.max_trajectory_length:
_observations = self.observations[-self.max_trajectory_length:]
@@ -256,7 +259,7 @@ class GPT4v_Agent:
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{_screenshot}",
"url": f"data:image/png;base64,{_screenshot}",
"detail": "high"
}
}
@@ -316,14 +319,14 @@ class GPT4v_Agent:
]
})
else:
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
messages.append({
"role": "assistant",
"content": [
{
"type": "text",
"text": previous_thought.stip() if len(previous_thought)>0 else "No valid action"
"text": previous_thought.strip() if len(previous_thought) > 0 else "No valid action"
},
]
})
@@ -437,7 +440,7 @@ class GPT4v_Agent:
]
})
else:
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
with open("messages.json", "w") as f:
f.write(json.dumps(messages, indent=4))

30
mm_agents/prompts.py
View File

@@ -3,8 +3,9 @@ You are an agent which follow my instruction and perform desktop computer tasks
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
For each step, you will get an observation of an image, which is the screenshot of the computer screen and you will predict the action of the computer based on the image.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -14,6 +15,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -267,8 +269,9 @@ You are an agent which follow my instruction and perform desktop computer tasks
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
For each step, you will get an observation of the desktop by accessibility tree, which is based on AT-SPI library. And you will predict the action of the computer based on the accessibility tree.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -278,6 +281,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -532,8 +536,9 @@ You have good knowledge of computer and good internet connection and assume your
For each step, you will get an observation of the desktop by 1) a screenshot; and 2) accessibility tree, which is based on AT-SPI library.
And you will predict the action of the computer based on the screenshot and accessibility tree.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -543,6 +548,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -797,7 +803,7 @@ You are an agent which follow my instruction and perform desktop computer tasks
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
For each step, you will get an observation of the desktop by 1) a screenshot; and 2) accessibility tree, which is based on AT-SPI library.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
You can replace x, y in the code with the tag of the element you want to operate with. such as:
```python
pyautogui.moveTo(tag#3)
@@ -806,7 +812,8 @@ pyautogui.dragTo(tag#1, button='left')
```
When you think you can directly output precise x and y coordinates or there is no tag on which you want to interact, you can also use them directly.
But you should be careful to ensure that the coordinates are correct.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -816,6 +823,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -845,7 +853,7 @@ Then, based on your analysis, in conjunction with human desktop using habits and
"""
ACTION_GROUNDING_PROMPT_SEEACT = """
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
You can replace x, y in the code with the tag of the element you want to operate with. such as:
```python
pyautogui.moveTo(tag#3)
@@ -854,7 +862,8 @@ pyautogui.dragTo(tag#1, button='left')
```
When you think you can directly output precise x and y coordinates or there is no tag on which you want to interact, you can also use them directly.
But you should be careful to ensure that the coordinates are correct.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -864,5 +873,6 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
"""