diff --git a/branch_flag b/branch_flag index 7e5437f..9daeafb 100644 --- a/branch_flag +++ b/branch_flag @@ -1 +1 @@ -baseline_som +test diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 9f57659..947fc92 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -9,20 +9,29 @@ from typing import List, Dict, Tuple import Xlib import lxml.etree -import pyatspi import pyautogui import requests from PIL import Image from Xlib import display, X from flask import Flask, request, jsonify, send_file, abort # , send_from_directory from lxml.etree import _Element -from pyatspi import Accessible, StateType, STATE_SHOWING -from pyatspi import Action as ATAction -from pyatspi import Component, Document -from pyatspi import Text as ATText -from pyatspi import Value as ATValue -from pywinauto import Desktop -from pywinauto.base_wrapper import BaseWrapper + +platform_name: str = platform.system() + +if platform_name=="Linux": + import pyatspi + from pyatspi import Accessible, StateType, STATE_SHOWING + from pyatspi import Action as ATAction + from pyatspi import Component, Document + from pyatspi import Text as ATText + from pyatspi import Value as ATValue + + BaseWrapper = Any +elif platform_name=="Windows": + from pywinauto import Desktop + from pywinauto.base_wrapper import BaseWrapper + + Accessible = Any from pyxcursor import Xcursor diff --git a/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json b/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json new file mode 100644 index 0000000..dfa3dcb --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json @@ -0,0 +1,76 @@ +{ + "id": "01b269ae-2111-4a07-81fd-3fcd711993b0", + "snapshot": "libreoffice_calc", + "instruction": "Fill all the blank cells with the value in the cell above it", + "source": "https://www.youtube.com/shorts/VrUzPTIwQ04", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032", + "path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx" + } + } + ], + "trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0", + "related_apps": [ + "msoffice_excel" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Student_Level_Fill_Blank.xlsx - Excel", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916", + "dest": "Student_Level_Fill_Blank_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx", + "dest": "Student_Level_Fill_Blank.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} diff --git a/main.py b/main.py index a3a01df..d971e8c 100644 --- a/main.py +++ b/main.py @@ -46,11 +46,11 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f: + with open("evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 30" + example["snapshot"] = "Snapshot 1" - env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" + env = DesktopEnv( path_to_vm="~/vmware/Windows 10 x64/Windows 10 x64.vmx" , action_space="computer_13" , task_config=example )