Fix minor bugs caused from merging in setupcontroller; Initialize vscode example loading

desktop_env/controllers/setup.py

@@ -20,6 +20,7 @@ class SetupController:
     def __init__(self, vm_ip: str, cache_dir: str):
         self.vm_ip: str = vm_ip
         self.http_server: str = f"http://{vm_ip}:5000"
+        self.http_server_setup_root: str = f"http://{vm_ip}:5000/setup"
         self.cache_dir: str = cache_dir
 
     def reset_cache_dir(self, cache_dir: str):

desktop_env/evaluators/getters/general.py

@@ -1,7 +1,5 @@
-from typing import Dict
-
 import os
-import requests
+from typing import Dict
 
 
 def get_string(env, config: Dict[str, str]) -> str:
@@ -12,6 +10,7 @@ def get_string(env, config: Dict[str, str]) -> str:
 
     return config["string"]
 
+
 def get_command_line(env, config: Dict[str, str]) -> str:
     """
     Config:

desktop_env/evaluators/getters/misc.py

@@ -1,6 +1,5 @@
 import logging
 from typing import TypeVar
-#from typing import Dict, List
 
 logger = logging.getLogger("desktopenv.getters.misc")
 
@@ -13,6 +12,7 @@ def get_rule(env, config: R) -> R:
     """
     return config["rules"]
 
+
 def get_accessibility_tree(env, *args) -> str:
     accessibility_tree: str = env.controller.get_accessibility_tree()
     logger.debug("AT@eval: %s", accessibility_tree)

desktop_env/evaluators/getters/replay.py

@@ -1,7 +1,7 @@
 from typing import List, Dict, Any
 
-def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
 
+def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
     def parse(action):
         if action["type"] == "hotkey":
             keys = "', '".join(action["param"])

desktop_env/evaluators/getters/vscode.py

@@ -1,10 +1,11 @@
-from typing import Dict
 from typing import Any
-from replay import get_replay
+from typing import Dict
-from file import get_vm_file
+
+from .file import get_vm_file
+from .replay import get_replay
+
 
 def get_vscode_config(env, config: Dict[str, Any]) -> str:
-
     trajectory = [{"type": "hotkey", "param": ["command", "shift", "p"]},
                   {"type": "typewrite", "param": "OpenProject"},
                   {"type": "press", "param": "enter"}]

desktop_env/evaluators/metrics/__init__.py

@@ -13,4 +13,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
 from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
 from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
-
+from .vscode import compare_text_file, compare_config, compare_answer

desktop_env/evaluators/metrics/docs.py

@@ -1,12 +1,14 @@
-import xml.etree.ElementTree as ET
+import logging
 import os
+import xml.etree.ElementTree as ET
 from typing import List, Dict, Any
+
 from docx import Document
 from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
 
-import logging
 logger = logging.getLogger("desktopenv.metric.docs")
 
+
 def find_default_font(config_file_path, rules):
     """Find the default font in LibreOffice Writer."""
     default_font = None

desktop_env/evaluators/metrics/libreoffice.py

@@ -1,18 +1,18 @@
-import lxml.cssselect
-from lxml.etree import _Element as Element
-import lxml.etree
 import fnmatch
-
 from typing import Dict, List
 
+import lxml.cssselect
+import lxml.etree
+from lxml.etree import _Element as Element
+
 _libconf_namespaces = [("oor", "http://openoffice.org/2001/registry")]
 _libconf_ns_mapping = dict(_libconf_namespaces)
-_setup_locale_selector = lxml.cssselect.CSSSelector( 'item[oor|path$=L10N]>prop[oor|name=ooSetupSystemLocale]>value'
+_setup_locale_selector = lxml.cssselect.CSSSelector('item[oor|path$=L10N]>prop[oor|name=ooSetupSystemLocale]>value',
-                                                   , namespaces=_libconf_ns_mapping
+                                                    namespaces=_libconf_ns_mapping)
-                                                   )
+_locale_selector = lxml.cssselect.CSSSelector('item[oor|path$=L10N]>prop[oor|name=ooLocale]>value',
-_locale_selector = lxml.cssselect.CSSSelector( 'item[oor|path$=L10N]>prop[oor|name=ooLocale]>value'
+                                              namespaces=_libconf_ns_mapping)
-                                             , namespaces=_libconf_ns_mapping
+
-                                             )
+
 def check_libre_locale(config_file: str, rules: Dict[str, List[str]]) -> float:
     config: Element = lxml.etree.parse(config_file).getroot()
     setup_locale_setting: List[Element] = _setup_locale_selector(config)
@@ -27,6 +27,7 @@ def check_libre_locale(config_file: str, rules: Dict[str, List[str]]) -> float:
                      )
                  )
 
+
 if __name__ == "__main__":
     path1 = "../../任务数据/LibreOffice Calc/registrymodifications.ru.xcu"
     print(check_libre_locale(path1, {"locale_set": ["ru-*", "de-*", "fr-*"

desktop_env/evaluators/metrics/pdf.py

@@ -1,13 +1,11 @@
-from pypdf import PdfReader
 import operator
-
-from typing import Dict
 from typing import Any
+from typing import Dict
+
+from pypdf import PdfReader
+
 
 def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float:
     reader = PdfReader(pdf_file)
     nb_pages: int = len(reader.pages)
-    return float( getattr(operator, rules["relation"])( nb_pages
+    return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"]))
-                                                      , rules["ref_value"]
-                                                      )
-                )

desktop_env/evaluators/metrics/table.py

@@ -1,18 +1,19 @@
-import pandas as pd
+import logging
+import operator
+from numbers import Number
+from typing import Any, Union
+from typing import Dict, List
+
 import openpyxl
+import pandas as pd
 from openpyxl import Workbook
 from openpyxl.worksheet.worksheet import Worksheet
 
 from .utils import load_charts, load_sparklines
-import operator
 
-from typing import Dict, List
-from typing import Any, Union
-from numbers import Number
-
-import logging
 logger = logging.getLogger("desktopenv.metric.table")
 
+
 def compare_table(actual: str, expected: str, **options) -> float:
     """
     Args:
@@ -73,6 +74,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
 
     return float(metric)
 
+
 def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
     if result is None:
         return 0.
@@ -114,6 +116,7 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
 
     return float(passes)
 
+
 def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
     if result is None:
         return 0.
@@ -121,6 +124,7 @@ def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
     worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
     return float(worksheet.freeze_panes == rules["position"])
 
+
 def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
     if result is None:
         return 0.
@@ -132,6 +136,7 @@ def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
                                                       )
                  )
 
+
 if __name__ == '__main__':
     # path1 = ""
     # path2 = ""

desktop_env/evaluators/metrics/vscode.py

@@ -1,3 +1,6 @@
+from typing import Dict
+
+
 def compare_text_file(actual: str, expected: str, **options) -> float:
     """
     Args:
@@ -17,24 +20,17 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
         return 1.0
     return 0.0
 
-def compare_config(actual: str, expected: str, **options) -> float:
-    """
-    Args:
-        actual (str): path to result text file
-        expected (str): gold string
-
-    Return:
-        float: the score
-    """
 
+def compare_config(actual: str, rules: Dict, **options) -> float:
     with open(actual) as f1:
         actual_text = f1.read()
 
-    if actual_text == expected:
+    if actual_text == rules['expect']:
         return 1.0
     return 0.0
 
-def compare_answer(actual: str, expected: str, **options) -> float:
+
+def compare_answer(actual: str, rules: Dict, **options) -> float:
     """
     Args:
         actual (str): result string
@@ -44,11 +40,8 @@ def compare_answer(actual: str, expected: str, **options) -> float:
         float: the score
     """
 
-    if actual == expected:
+    if actual == rules['expect']:
         return 1.0
 
     # TODO: can use text embedding to get non-zero return
     return 0.0
-
-if __name__ == '__main__':
-    print(compare_text_file("README.md", "README.md"))

evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json

@@ -16,9 +16,9 @@
       }
     },
     {
-      "type": "open",
+      "type": "launch",
       "parameters": {
-        "path": "Desktop/vscode_replace_text.txt"
+        "command": ["code", "Desktop/vscode_replace_text.txt"]
       }
     }
   ],

evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json

@@ -4,6 +4,12 @@
   "instruction": "Could you help me open the project at /home/user/project?",
   "source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
   "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": ["code"]
+      }
+    },
     {
       "type": "command",
       "parameters": {
@@ -18,8 +24,10 @@
   "evaluator": {
     "func": "compare_config",
     "expected": {
-      "type": "string",
+      "type": "rule",
-      "string": "project"
+      "rules": {
+        "expect": "project"
+      }
     },
     "result": {
       "type": "vscode_config",

evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json

@@ -29,8 +29,10 @@
   "evaluator": {
     "func": "compare_config",
     "expected": {
-      "type": "string",
+      "type": "rule",
-      "string": "100"
+      "rules": {
+        "expect": "100"
+      }
     },
     "result": {
       "type": "vscode_config",