lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(env): handle None result_getter for vllm_eval evaluator

Browse Source
This commit is contained in:
cui0711
2026-02-09 17:46:05 +08:00
parent 3890ee5fc3
commit ba03784196
1 changed files with 22 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
desktop_env/desktop_env.py
View File

@@ -474,13 +474,18 @@ class DesktopEnv(gym.Env):
if "expected" in self.evaluator: if "expected" in self.evaluator:
assert len(self.metric) == len(self.expected_getter), "The number of metrics and expected getters must be the same" assert len(self.metric) == len(self.expected_getter), "The number of metrics and expected getters must be the same"
for idx, metric in enumerate(self.metric): for idx, metric in enumerate(self.metric):
try: # Skip result state extraction if result_getter is None (e.g., for vllm_eval)
config = self.evaluator["result"][idx] if self.result_getter[idx] is not None:
result_state = self.result_getter[idx](self, config) try:
except FileNotFoundError: config = self.evaluator["result"][idx]
logger.error("File not found!") result_state = self.result_getter[idx](self, config)
if self.metric_conj == 'and': except FileNotFoundError:
return 0 logger.error("File not found!")
if self.metric_conj == 'and':
return 0
else:
# For evaluators that don't need result state (e.g., vllm_eval)
result_state = None
if "expected" in self.evaluator and self.expected_getter and self.evaluator["expected"]: if "expected" in self.evaluator and self.expected_getter and self.evaluator["expected"]:
expected_state = self.expected_getter[idx](self, self.evaluator["expected"][idx]) expected_state = self.expected_getter[idx](self, self.evaluator["expected"][idx])
@@ -498,11 +503,16 @@ class DesktopEnv(gym.Env):
return sum(results) / len(results) if self.metric_conj == 'and' else max(results) return sum(results) / len(results) if self.metric_conj == 'and' else max(results)
else: else:
# Single metric to evaluate whether the task is successfully completed # Single metric to evaluate whether the task is successfully completed
try: # For evaluators like vllm_eval that don't need result_getter, skip result state extraction
result_state = self.result_getter(self, self.evaluator["result"]) if self.result_getter is not None:
except FileNotFoundError: try:
logger.error("File not found!") result_state = self.result_getter(self, self.evaluator["result"])
return 0 except FileNotFoundError:
logger.error("File not found!")
return 0
else:
# For evaluators that don't need result state (e.g., vllm_eval)
result_state = None
if "expected" in self.evaluator and self.expected_getter and self.evaluator["expected"]: if "expected" in self.evaluator and self.expected_getter and self.evaluator["expected"]:
expected_state = self.expected_getter(self, self.evaluator["expected"]) expected_state = self.expected_getter(self, self.evaluator["expected"])