lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update AWS AMI ID, enhance directory creation logic in file upload, modify osworld service configuration, and refine JSON evaluation examples for improved clarity and functionality.

Browse Source
This commit is contained in:
yuanmengqi
2025-07-03 16:57:41 +00:00
parent 7b2120c843
commit a651b04e49
5 changed files with 55 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation_examples/examples/os/37887e8c-da15-4192-923c-08fa390a176d.json
View File

@@ -57,7 +57,7 @@
"parameters": {
"files": [
{
"url": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/os/37887e8c-da15-4192-923c-08fa390a176d/eval.sh",
"url": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/os/37887e8c-da15-4192-923c-08fa390a176d/eval_20250703.sh",
"path": "eval.sh"
}
]

60
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -20,25 +20,49 @@
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "LARGE_TEXT=$(gsettings get org.gnome.desktop.interface text-scaling-factor 2>/dev/null | grep -E '1\\.[2-9]|[2-9]\\.' | wc -l) && ZOOM_ENABLED=$(gsettings get org.gnome.desktop.a11y.applications screen-magnifier-enabled 2>/dev/null | grep -c true) && ZOOM_FACTOR=$(gsettings get org.gnome.desktop.a11y.magnifier mag-factor 2>/dev/null | grep -E '[2-9]\\.|1\\.[1-9]' | wc -l) && if [ \"$LARGE_TEXT\" -gt 0 ]; then echo \"Large text enabled\"; elif [ \"$ZOOM_ENABLED\" -gt 0 ] && [ \"$ZOOM_FACTOR\" -gt 0 ]; then echo \"Zoom enabled with magnification\"; else echo \"No enlargement enabled\"; fi",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"include": [
"Large text enabled",
"Zoom enabled with magnification"
],
"exclude": [
"No enlargement enabled",
"error"
]
"func": [
"check_include_exclude",
"check_include_exclude"
],
"conj": "or",
"result": [
{
"type": "vm_command_line",
"command": "TEXT_SCALE=$(timeout 10 gsettings get org.gnome.desktop.interface text-scaling-factor 2>/dev/null | grep -oE '[0-9]+\\.[0-9]+' | head -1); if [ -n \"$TEXT_SCALE\" ] && [ \"$(echo \"$TEXT_SCALE >= 1.2\" | bc -l 2>/dev/null || echo 0)\" = \"1\" ]; then echo \"Large text enabled (factor: $TEXT_SCALE)\"; else echo \"No text scaling enabled\"; fi",
"shell": true
},
{
"type": "vm_command_line",
"command": "ZOOM_ENABLED=$(timeout 10 gsettings get org.gnome.desktop.a11y.applications screen-magnifier-enabled 2>/dev/null | grep -c true); ZOOM_FACTOR=$(timeout 10 gsettings get org.gnome.desktop.a11y.magnifier mag-factor 2>/dev/null | grep -oE '[0-9]+\\.[0-9]+' | head -1); if [ \"$ZOOM_ENABLED\" -gt 0 ] && [ -n \"$ZOOM_FACTOR\" ] && [ \"$(echo \"$ZOOM_FACTOR > 1.1\" | bc -l 2>/dev/null || echo 0)\" = \"1\" ]; then echo \"Zoom enabled with magnification (factor: $ZOOM_FACTOR)\"; else echo \"No zoom magnification enabled\"; fi",
"shell": true
}
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"Large text enabled"
],
"exclude": [
"No text scaling enabled",
"error"
]
}
},
{
"type": "rule",
"rules": {
"include": [
"Zoom enabled with magnification"
],
"exclude": [
"No zoom magnification enabled",
"error"
]
}
}
]
},
"proxy": false
}