ver Jan30thv2

debugged on windows platform with new _create_pywinauto_node function
migrated example task from calc to excel

branch_flag

@@ -1 +1 @@
-baseline_som
+test

desktop_env/server/main.py

@@ -9,20 +9,29 @@ from typing import List, Dict, Tuple
 
 import Xlib
 import lxml.etree
-import pyatspi
 import pyautogui
 import requests
 from PIL import Image
 from Xlib import display, X
 from flask import Flask, request, jsonify, send_file, abort  # , send_from_directory
 from lxml.etree import _Element
-from pyatspi import Accessible, StateType, STATE_SHOWING
+
-from pyatspi import Action as ATAction
+platform_name: str = platform.system()
-from pyatspi import Component, Document
+
-from pyatspi import Text as ATText
+if platform_name=="Linux":
-from pyatspi import Value as ATValue
+    import pyatspi
-from pywinauto import Desktop
+    from pyatspi import Accessible, StateType, STATE_SHOWING
-from pywinauto.base_wrapper import BaseWrapper
+    from pyatspi import Action as ATAction
+    from pyatspi import Component, Document
+    from pyatspi import Text as ATText
+    from pyatspi import Value as ATValue
+
+    BaseWrapper = Any
+elif platform_name=="Windows":
+    from pywinauto import Desktop
+    from pywinauto.base_wrapper import BaseWrapper
+
+    Accessible = Any
 
 from pyxcursor import Xcursor
 

evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json

@@ -0,0 +1,76 @@
+{
+	"id": "01b269ae-2111-4a07-81fd-3fcd711993b0",
+	"snapshot": "libreoffice_calc",
+	"instruction": "Fill all the blank cells with the value in the cell above it",
+	"source": "https://www.youtube.com/shorts/VrUzPTIwQ04",
+	"config": [
+		{
+			"type": "download",
+			"parameters": {
+				"files": [
+					{
+						"url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032",
+						"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
+					}
+				]
+			}
+		},
+		{
+			"type": "open",
+			"parameters": {
+				"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
+			}
+		}
+	],
+	"trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0",
+	"related_apps": [
+		"msoffice_excel"
+	],
+	"evaluator": {
+		"postconfig": [
+			{
+				"type": "activate_window",
+				"parameters": {
+					"window_name": "Student_Level_Fill_Blank.xlsx - Excel",
+					"strict": true
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python",
+						"-c",
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
+					]
+				}
+			}
+		],
+		"func": "compare_table",
+		"expected": {
+			"type": "cloud_file",
+			"path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916",
+			"dest": "Student_Level_Fill_Blank_gold.xlsx"
+		},
+		"result": {
+			"type": "vm_file",
+			"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx",
+			"dest": "Student_Level_Fill_Blank.xlsx"
+		},
+		"options": {
+			"rules": [
+				{
+					"type": "sheet_data",
+					"sheet_idx0": 0,
+					"sheet_idx1": "EI0"
+				}
+			]
+		}
+	}
+}

main.py

@@ -46,11 +46,11 @@ def human_agent():
     Runs the Gym environment with human input.
     """
 
-    with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f:
+    with open("evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json", "r") as f:
         example = json.load(f)
-    example["snapshot"] = "Snapshot 30"
+    example["snapshot"] = "Snapshot 1"
 
-    env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
+    env = DesktopEnv( path_to_vm="~/vmware/Windows 10 x64/Windows 10 x64.vmx"
                     , action_space="computer_13"
                     , task_config=example
                     )