lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan30thv2

Browse Source 
debugged on windows platform with new _create_pywinauto_node function
migrated example task from calc to excel
This commit is contained in:
David Chang
2024-01-30 21:09:53 +08:00
parent da306376da
commit 14dbc708a4
4 changed files with 97 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
branch_flag
View File

@@ -1 +1 @@
baseline_som
test

25
desktop_env/server/main.py
View File

@@ -9,20 +9,29 @@ from typing import List, Dict, Tuple
import Xlib
import lxml.etree
import pyatspi
import pyautogui
import requests
from PIL import Image
from Xlib import display, X
from flask import Flask, request, jsonify, send_file, abort # , send_from_directory
from lxml.etree import _Element
from pyatspi import Accessible, StateType, STATE_SHOWING
from pyatspi import Action as ATAction
from pyatspi import Component, Document
from pyatspi import Text as ATText
from pyatspi import Value as ATValue
from pywinauto import Desktop
from pywinauto.base_wrapper import BaseWrapper
platform_name: str = platform.system()
if platform_name=="Linux":
import pyatspi
from pyatspi import Accessible, StateType, STATE_SHOWING
from pyatspi import Action as ATAction
from pyatspi import Component, Document
from pyatspi import Text as ATText
from pyatspi import Value as ATValue
BaseWrapper = Any
elif platform_name=="Windows":
from pywinauto import Desktop
from pywinauto.base_wrapper import BaseWrapper
Accessible = Any
from pyxcursor import Xcursor

76
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json Normal file
View File

@@ -0,0 +1,76 @@
{
"id": "01b269ae-2111-4a07-81fd-3fcd711993b0",
"snapshot": "libreoffice_calc",
"instruction": "Fill all the blank cells with the value in the cell above it",
"source": "https://www.youtube.com/shorts/VrUzPTIwQ04",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032",
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
}
}
],
"trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0",
"related_apps": [
"msoffice_excel"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Student_Level_Fill_Blank.xlsx - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916",
"dest": "Student_Level_Fill_Blank_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx",
"dest": "Student_Level_Fill_Blank.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

6
main.py
View File

@@ -46,11 +46,11 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f:
with open("evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json", "r") as f:
example = json.load(f)
example["snapshot"] = "Snapshot 30"
example["snapshot"] = "Snapshot 1"
env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
env = DesktopEnv( path_to_vm="~/vmware/Windows 10 x64/Windows 10 x64.vmx"
, action_space="computer_13"
, task_config=example
)