Merge branch 'zdy'

.gitignore

@@ -163,6 +163,7 @@ frontend/.next/
 frontend/.idea
 
 tags
+tags-opts
 snapshots
 branch_flag
 branch-config

desktop_env/controllers/setup.py

@@ -1,13 +1,23 @@
 import requests
 import json
+from requests_toolbelt.multipart.encoder import MultipartEncoder
+
+import uuid
+import os.path
 
 from typing import Dict, List
 from typing import Any
 
 
 class SetupController:
-    def __init__(self, http_server: str):
+    def __init__( self
+                , http_server: str
+                , cache_dir: str
+                ):
         self.http_server = http_server + "/setup"
+        self.cache_dir: str = cache_dir
+    def reset_cache_dir(self, cache_dir: str):
+        self.cache_dir = cache_dir
 
     def setup(self, config: List[Dict[str, Any]]):
         """
@@ -56,22 +66,56 @@ class SetupController:
         for f in files:
             url: str = f["url"]
             path: str = f["path"]
+            cache_path: str = os.path.join( self.cache_dir
+                                          , "{:}_{:}".format(
+                                              uuid.uuid5(uuid.NAMESPACE_URL, url)
+                                            , os.path.basename(path)
+                                            )
+                                          )
 
             if not url or not path:
                 raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
 
-            payload = json.dumps({"url": url, "path": path})
+            if not os.path.exists(cache_path):
-            headers = {
+                max_retries = 3
-                'Content-Type': 'application/json'
+                downloaded = False
-            }
+                for i in range(max_retries):
+                    try:
+                        response = requests.get(url, stream=True)
+                        response.raise_for_status()
 
-            # send request to server to download file
+                        with open(cache_path, 'wb') as f:
+                            for chunk in response.iter_content(chunk_size=8192):
+                                if chunk:
+                                    f.write(chunk)
+                        print("File downloaded successfully")
+                        downloaded = True
+                        break
+
+                    except requests.RequestException as e:
+                        print(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)")
+                if not downloaded:
+                    raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
+
+            #payload = json.dumps({"url": url, "path": path})
+            #headers = {
+                #'Content-Type': 'application/json'
+            #}
+
+            form = MultipartEncoder( { "file_path": path
+                                     , "file_data": (os.path.basename(path), open(cache_path, "rb"))
+                                     }
+                                   )
+            headers = {"Content-Type": form.content_type}
+            print(form.content_type)
+
+            # send request to server to upload file
             try:
-                response = requests.post(self.http_server + "/download_file", headers=headers, data=payload)
+                response = requests.post(self.http_server + "/upload", headers=headers, data=form)
                 if response.status_code == 200:
                     print("Command executed successfully:", response.text)
                 else:
-                    print("Failed to download file. Status code:", response.text)
+                    print("Failed to upload file. Status code:", response.text)
             except requests.exceptions.RequestException as e:
                 print("An error occurred while trying to send the request:", e)
 

desktop_env/envs/desktop_env.py

@@ -67,18 +67,6 @@ class DesktopEnv(gym.Env):
         self.tmp_dir_base: str = tmp_dir
         self.cache_dir_base: str = cache_dir
 
-        # Initialize emulator and controller
-        print("Initializing...")
-        self._start_emulator()
-        self.host = f"http://{self._get_vm_ip()}:5000"
-        self.controller = PythonController(http_server=self.host)
-        self.setup_controller = SetupController(http_server=self.host)
-
-        # mode: human or machine
-        assert action_space in ["computer_13", "pyautogui"]
-        self.action_space = action_space
-        # todo: define the action space and the observation space as gym did, or extend theirs
-
         # task-aware stuffs
         self.snapshot_path = task_config["snapshot"]  # todo: handling the logic of snapshot directory
         self.task_id: str = task_config["id"]
@@ -92,10 +80,24 @@ class DesktopEnv(gym.Env):
         self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
         self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
 
-        # episodic stuffs, like tmp dir and counters
+        # Initialize emulator and controller
-        self.tmp_dir: str = self.tmp_dir_base  # just an init value, updated during reset
+        print("Initializing...")
+        self._start_emulator()
+        self.host = f"http://{self._get_vm_ip()}:5000"
+        self.controller = PythonController(http_server=self.host)
+        self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir)
+
+        # mode: human or machine
+        assert action_space in ["computer_13", "pyautogui"]
+        self.action_space = action_space
+        # todo: define the action space and the observation space as gym did, or extend theirs
+
+        # episodic stuffs, like tmp dir and counters, will be updated or reset
+        # when calling self.reset()
+        self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset
         self._traj_no: int = -1
         self._step_no: int = 0
+        self.action_history: List[Dict[str, any]] = []
 
     def _start_emulator(self):
         while True:
@@ -164,9 +166,12 @@ class DesktopEnv(gym.Env):
             self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
             self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
 
+            self.setup_controller.reset_cache_dir(self.cache_dir)
+
         print("Setting counters...")
         self._traj_no += 1
         self._step_no = 0
+        self.action_history.clear()
 
         print("Setup new temp dir...")
         self.tmp_dir = tempfile.mkdtemp(
@@ -202,6 +207,7 @@ class DesktopEnv(gym.Env):
         elif self.action_space == "pyautogui":
             # the set of all possible python commands insides `pyautogui`
             self.controller.execute_python_command(action)
+        self.action_history.append(action)
 
         # todo: maybe for the better here we need to add a logic to wait until the rendering is done
         time.sleep(pause)

desktop_env/evaluators/metrics/__init__.py

@@ -1 +1 @@
-from .table import compare_table
+from .table import compare_table, compare_with_sparklines

desktop_env/evaluators/metrics/table.py

@@ -1,14 +1,74 @@
-def compare_table(expected, actual):
+import pandas as pd
-    import pandas as pd
+import zipfile
+import lxml.etree
+import lxml.cssselect
+from lxml.etree import _Element
+import xmltodict
+#import pylightxl
+
+from typing import Dict, List
+#from typing import Any
+
+def compare_table(actual, expected):
     df1 = pd.read_excel(expected)
     df2 = pd.read_excel(actual)
 
     # Compare the DataFrames
     return 1 if df1.equals(df2) else 0
 
+_xlsx_namespaces = [ ("x14", "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main")
+                   , ("xm", "http://schemas.microsoft.com/office/excel/2006/main")
+                   ]
+_xlsx_ns_mapping = dict(_xlsx_namespaces)
+_xlsx_ns_imapping = dict(map(lambda itm: (itm[1], itm[0]), _xlsx_namespaces))
+_sparklines_selector = lxml.cssselect.CSSSelector("x14|sparkline", namespaces=_xlsx_ns_mapping)
+#print(_sparklines_selector.css)
+def _load_sparklines(xlsx_file: str) -> Dict[str, str]:
+    """
+    This function modifies data_frame in-place
+
+    Args:
+        xlsx_file (str): path to xlsx
+
+    Returns:
+        List[Dict[str, str]]: sparkline definitions in form of
+          {
+            "F3": "Sheet1!C3:E3"
+          }
+    """
+
+    # read xlsx
+    with zipfile.ZipFile(xlsx_file, "r") as z_f:
+        with z_f.open("xl/worksheets/sheet1.xml") as f:
+            sheet1: _Element = lxml.etree.fromstring(f.read())
+            sparklines: List[_Element] = _sparklines_selector(sheet1)
+            
+    sparklines_dict: Dict[str, str] = {}
+    for sp_l in sparklines:
+        sparkline_xml: str = lxml.etree.tostring(sp_l, encoding="unicode")
+        sparkline: Dict[str, Dict[str, str]] = xmltodict.parse( sparkline_xml
+                                                              , process_namespaces=True
+                                                              , namespaces=_xlsx_ns_imapping
+                                                              )
+        sparklines_dict[sparkline["x14:sparkline"]["xm:sqref"]] = sparkline["x14:sparkline"]["xm:f"]
+    return sparklines_dict
+
+def compare_with_sparklines(actual: str, expected: str) -> float:
+    df1 = pd.read_excel(actual)
+    df2 = pd.read_excel(expected)
+    normal_content_metric: bool = df1.equals(df2)
+
+    sp1 = _load_sparklines(actual)
+    sp2 = _load_sparklines(expected)
+    sparkline_metric: bool = sp1 == sp2
+
+    return float(normal_content_metric and sparkline_metric)
 
 if __name__ == '__main__':
-    path1 = ""
+    #path1 = ""
-    path2 = ""
+    #path2 = ""
-
+    #print(compare_table(path1, path2))
-    print(compare_table(path1, path2))
+    
+    path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx"
+    path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx"
+    print(compare_with_sparklines(path1, path2))

desktop_env/server/main.py

@@ -95,6 +95,16 @@ def get_file():
         # If the file is not found, return a 404 error
         return jsonify({"error": "File not found"}), 404
 
+@app.route("/setup/upload", methods=["POST"])
+def upload_file():
+    # Retrieve filename from the POST request
+    if 'file_path' in request.form and 'file_data' in request.files:
+        file_path = request.form['file_path']
+        file = request.files["file_data"]
+        file.save(file_path)
+        return "File Uploaded"
+    else:
+        return jsonify({"error": "file_path and file_data are required"}), 400
 
 @app.route('/platform', methods=['GET'])
 def get_platform():

evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json

@@ -0,0 +1,57 @@
+{
+  "id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
+  "snapshot": "libreoffice_calc",
+  "instruction": "Copy sheet1 and insert it before sheet2",
+  "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1ejNXBNOZtn64ugmvXot21pOEjx5xa-I5&export=download&authuser=0&confirm=t&uuid=61aa93e2-03f7-4b28-8e4a-cdff16a642f7&at=APZUnTVgPAHHfXaEjfKau5CDY1_K:1703509323791",
+            "path": "Desktop/copy_sheet_insert.xlsx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "Desktop/copy_sheet_insert.xlsx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
+  "related_apps": [
+    "libreoffice_calc"
+  ],
+  "evaluator": {
+    "func": "check_sheet_list",
+    "result": {
+      "type": "vm_file",
+      "path": "Desktop/copy_sheet_insert.xlsx",
+      "dest": "copy_sheet_insert.xlsx"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": [
+        {
+          "type": "sheet_name",
+          "sheet_idx": 0,
+          "sheet_name": "Sheet1"
+        },
+        {
+          "type": "sheet_data",
+          "sheet_idx0": 0,
+          "sheet_idx1": 1
+        },
+        {
+          "type": "sheet_name",
+          "sheet_idx": 2,
+          "sheet_name": "Sheet2"
+        }
+      ]
+    }
+  }
+}

evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json

@@ -3,20 +3,38 @@
   "snapshot": "libreoffice_calc",
   "instruction": "Make sparkline chart line by line",
   "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY",
-  "config": {
+  "config": [
-    "download": [
+    {
-      [
+      "type": "download",
-        "",
+      "parameters": {
-        "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx"
+        "files": [
-      ]
+          {
-    ],
+            "url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351",
-    "open": [
+            "path": "Desktop/OrderId_Month_Chart.xlsx"
-      "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "Desktop/OrderId_Month_Chart.xlsx"
     ]
   },
   "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
   "related_apps": [
     "libreoffice calc"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094",
+      "dest": "OrderId_Month_Chart_gold.xlsx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "Desktop/OrderId_Month_Chart.xlsx",
+      "dest": "OrderId_Month_Chart.xlsx"
+    }
+  }
 }

evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json

@@ -3,32 +3,40 @@
   "snapshot": "libreoffice_calc",
   "instruction": "Help me fill the columns of First Name, Last Name and Rank",
   "source": "https://www.youtube.com/shorts/uzPo_CPCHH8",
-  "config": {
+  "config": [
-    "download": [
+    {
-      [
+      "type": "download",
-        "https://drive.usercontent.google.com/download?id=1wDqap5cBfxnlqTNrZG61k_wDWTujl6AU&export=download&authuser=0&confirm=t&uuid=fd183b89-76b7-4dc5-880e-1045ed769562&at=APZUnTWp9RMafMg0xohhBWazN3YD:1701785710674",
+      "parameters": {
-        "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx"
+        "files": [
-      ]
+          {
-    ],
+            "url": "",
-    "open": [
+            "path": "Desktop/Employee_Roles_and_Ranks.xlsx"
-      "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx"
+          }
-    ]
+        ]
-  },
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "Desktop/Employee_Roles_and_Ranks.xlsx"
+      }
+    }
+  ],
   "trajectory": "trajectories/37608790-6147-45d0-9f20-1137bb35703d",
   "related_apps": [
     "libreoffice calc"
   ],
   "evaluator": {
-    "func": "compare_table(expected, actual)",
+    "func": "compare_table",
-    "paths": {
+    "expected": {
-      "expected": {
+      "type": "cloud_file",
-        "type": "cloud_file",
+      "path": "",
-        "path": "https://drive.usercontent.google.com/download?id=1dxpiUqP_CVvQp5tddxlwO3Cp1BqJ-ZDE&export=download&authuser=0&confirm=t&uuid=ccd204c7-07ce-4fdf-a5d4-a7e4f37b9ce6&at=APZUnTVBs7TgrVrDXpkiU8S7WbQo:1702360836747"
+      "dest": "Employee_Roles_and_Ranks_gold.xlsx"
-      },
+    },
-      "actual": {
+    "result": {
-        "type": "vm_file",
+      "type": "vm_file",
-        "path": "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx"
+      "path": "Desktop/Employee_Roles_and_Ranks.xlsx",
-      }
+      "dest": "Employee_Roles_and_Ranks.xlsx"
     }
   }
 }

requirements.txt

@@ -13,4 +13,8 @@ pyautogui~=0.9.54
 psutil~=5.9.6
 tqdm~=4.65.0
 pandas~=2.0.3
-flask~=3.0.0
+flask~=3.0.0
+requests-toolbelt~=1.0.0
+lxml
+cssselect
+xmltodict