lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Loading libreoffice writer examples and find few problems, will do another round tomorrow for the rest

Browse Source
This commit is contained in:
Timothyxxx
2024-01-02 17:50:05 +08:00
parent 8ac88e9617
commit 03e99a68fb
22 changed files with 191 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
desktop_env/envs/desktop_env.py
View File

@@ -78,7 +78,8 @@ class DesktopEnv(gym.Env):
self.evaluator = task_config["evaluator"]
self.metric: Metric = getattr(metrics, self.evaluator["func"])
self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(
self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None
self.metric_options: Dict[str, Any] = self.evaluator.get("options", {})
# Initialize emulator and controller
@@ -95,7 +96,7 @@ class DesktopEnv(gym.Env):
# episodic stuffs, like tmp dir and counters, will be updated or reset
# when calling self.reset()
self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset
self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset
self._traj_no: int = -1
self._step_no: int = 0
self.action_history: List[Dict[str, any]] = []
@@ -165,7 +166,8 @@ class DesktopEnv(gym.Env):
self.evaluator = task_config["evaluator"]
self.metric: Metric = getattr(metrics, self.evaluator["func"])
self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(
self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None
self.metric_options = self.evaluator.get("options", {})
self.setup_controller.reset_cache_dir(self.cache_dir)
@@ -226,20 +228,12 @@ class DesktopEnv(gym.Env):
"""
Evaluate whether the task is successfully completed.
"""
result_state = self.result_getter(self, self.evaluator["result"])
expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \
else None
# todo: make this more flexible by refactoring
# eval_func = eval_funcs[self.evaluator["func"]]
# eval_func_vars = {}
#
# for var_name, file_info in self.evaluator["paths"].items():
# path = copy_file_to_local(file_info)
# eval_func_vars[var_name] = path
#
# return eval_func(**eval_func_vars)
result = self.result_getter(self, self.evaluator["result"])
expected = self.expected_getter(self, self.evaluator["expected"])
metric: float = self.metric(result, expected, **self.metric_options)
metric: float = self.metric(result_state, expected_state, **self.metric_options) if expected_state is not None \
else self.metric(result_state, **self.metric_options)
return metric