lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into fix_chrome

Browse Source
This commit is contained in:
yuanmengqi
2025-07-11 12:16:23 +00:00
parent fe3bb2fd92 6f0382c0c2
commit 37c56533f0
31 changed files with 3853 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
evaluation_examples/examples/libreoffice_impress/04578141-1d42-4146-b9cf-6fab4ce5fd74.json
View File

@@ -1,7 +1,7 @@
{
"id": "04578141-1d42-4146-b9cf-6fab4ce5fd74",
"snapshot": "libreoffice_impress",
"instruction":"Color the first three textboxes on slide 1 yellow, red, and green, respectively, in top-to-bottom order. Use exactly these colors—no variations (e.g., no dark red, light green, etc.).",
"instruction": "Color the first three textboxes on slide 1 yellow, red, and green, respectively, in top-to-bottom order. Use exactly these colors—no variations (e.g., no dark red, light green, etc.).",
"source": "https://arxiv.org/pdf/2311.01767.pdf",
"config": [
{
@@ -64,18 +64,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/04578141-1d42-4146-b9cf-6fab4ce5fd74/45_2_Gold.pptx",
"dest": "45_2_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/45_2.pptx",
"dest": "45_2.pptx"
},
"options": {}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/04578141-1d42-4146-b9cf-6fab4ce5fd74/45_2_Gold.pptx",
"dest": "45_2_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/04578141-1d42-4146-b9cf-6fab4ce5fd74/45_2_Gold_all_fonts.pptx",
"dest": "45_2_Gold_All_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/45_2.pptx",
"dest": "45_2.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/45_2.pptx",
"dest": "45_2.pptx"
}
]
},
"proxy": false
}

35
evaluation_examples/examples/libreoffice_impress/05dd4c1d-c489-4c85-8389-a7836c4f0567.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/05dd4c1d-c489-4c85-8389-a7836c4f0567/38_1_Gold.pptx",
"dest": "38_1_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/38_1.pptx",
"dest": "38_1.pptx"
},
"options": {}
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/05dd4c1d-c489-4c85-8389-a7836c4f0567/38_1_Gold_all_fonts.pptx",
"dest": "38_1_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/38_1.pptx",
"dest": "38_1.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/38_1.pptx",
"dest": "38_1.pptx"
}
]
},
"proxy": false
}

41
evaluation_examples/examples/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302/22_6_Gold.pptx",
"dest": "22_6_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/22_6.pptx",
"dest": "22_6.pptx"
},
"options": {}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302/22_6_Gold.pptx",
"dest": "22_6_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302/22_6_Gold2.pptx",
"dest": "22_6_Gold2.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/22_6.pptx",
"dest": "22_6.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/22_6.pptx",
"dest": "22_6.pptx"
}
]
},
"proxy": false
}

3
evaluation_examples/examples/libreoffice_impress/3161d64e-3120-47b4-aaad-6a764a92493b.json
View File

@@ -70,8 +70,7 @@
"dest": "45_1.pptx"
},
"options": {
"examine_shape": false,
"examine_shape_for_shift_size": true
"examine_shape": false
}
},
"proxy": false

43
evaluation_examples/examples/libreoffice_impress/4ed5abd0-8b5d-47bd-839f-cacfa15ca37a.json
View File

@@ -58,20 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/4ed5abd0-8b5d-47bd-839f-cacfa15ca37a/4_1_Gold.pptx",
"dest": "4_1_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/4_1.pptx",
"dest": "4_1.pptx"
},
"options": {
"examine_shape": false
}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/4ed5abd0-8b5d-47bd-839f-cacfa15ca37a/4_1_Gold.pptx",
"dest": "4_1_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/4ed5abd0-8b5d-47bd-839f-cacfa15ca37a/4_1_Gold_all_fonts.pptx",
"dest": "4_1_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/4_1.pptx",
"dest": "4_1.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/4_1.pptx",
"dest": "4_1.pptx"
}
]
},
"proxy": false
}

40
evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
View File

@@ -74,17 +74,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54/New_Club_Spring_2018_Training_with_strike.data",
"dest": "New_Club_Spring_2018_Training_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/New_Club_Spring_2018_Training.pptx",
"dest": "New_Club_Spring_2018_Training.pptx"
}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54/New_Club_Spring_2018_Training_with_strike.data",
"dest": "New_Club_Spring_2018_Training_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54/New_Club_Spring_2018_Training_Gold_all_fonts.pptx",
"dest": "New_Club_Spring_2018_Training_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/New_Club_Spring_2018_Training.pptx",
"dest": "New_Club_Spring_2018_Training.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/New_Club_Spring_2018_Training.pptx",
"dest": "New_Club_Spring_2018_Training.pptx"
}
]
},
"proxy": false
}

41
evaluation_examples/examples/libreoffice_impress/57667013-ea97-417c-9dce-2713091e6e2a.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/57667013-ea97-417c-9dce-2713091e6e2a/1_2_Gold.pptx",
"dest": "1_2_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/1_2.pptx",
"dest": "1_2.pptx"
},
"options": {}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/57667013-ea97-417c-9dce-2713091e6e2a/1_2_Gold.pptx",
"dest": "1_2_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/57667013-ea97-417c-9dce-2713091e6e2a/1_2_Gold_all_fonts.pptx",
"dest": "1_2_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/1_2.pptx",
"dest": "1_2.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/1_2.pptx",
"dest": "1_2.pptx"
}
]
},
"proxy": false
}

41
evaluation_examples/examples/libreoffice_impress/5c1a6c3d-c1b3-47cb-9b01-8d1b7544ffa1.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/5c1a6c3d-c1b3-47cb-9b01-8d1b7544ffa1/39_2_Gold.pptx",
"dest": "39_2_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/39_2.pptx",
"dest": "39_2.pptx"
},
"options": {}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/5c1a6c3d-c1b3-47cb-9b01-8d1b7544ffa1/39_2_Gold.pptx",
"dest": "39_2_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/5c1a6c3d-c1b3-47cb-9b01-8d1b7544ffa1/39_2_Gold_all_fonts.pptx",
"dest": "39_2_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/39_2.pptx",
"dest": "39_2.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/39_2.pptx",
"dest": "39_2.pptx"
}
]
},
"proxy": false
}

35
evaluation_examples/examples/libreoffice_impress/5cfb9197-e72b-454b-900e-c06b0c802b40.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/5cfb9197-e72b-454b-900e-c06b0c802b40/33_1_Gold.pptx",
"dest": "33_1_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/33_1.pptx",
"dest": "33_1.pptx"
},
"options": {}
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/5cfb9197-e72b-454b-900e-c06b0c802b40/33_1_Gold_all_fonts.pptx",
"dest": "33_1_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/33_1.pptx",
"dest": "33_1.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/33_1.pptx",
"dest": "33_1.pptx"
}
]
},
"proxy": false
}

59
evaluation_examples/examples/libreoffice_impress/7ae48c60-f143-4119-b659-15b8f485eb9a.json
View File

@@ -1,7 +1,7 @@
{
"id": "7ae48c60-f143-4119-b659-15b8f485eb9a",
"snapshot": "libreoffice_impress",
"instruction": "Change picture's height to 20,30,25cm on slide 3,4,6 respectively.",
"instruction": "Change picture's height to 20, 30, 25cm on slide 3, 4, 6 respectively.",
"hint": "The input unit is cm",
"source": "https://arxiv.org/pdf/2311.01767.pdf",
"config": [
@@ -28,6 +28,7 @@
"libreoffice_impress"
],
"evaluator": {
"_config_note": "CRITICAL: examine_shape must be False for examine_modify_height to work. Due to execution order, examine_shape check happens before examine_modify_height, so any shape dimension mismatch will cause immediate failure before examine_modify_height logic is reached.",
"postconfig": [
{
"type": "activate_window",
@@ -59,21 +60,47 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/7ae48c60-f143-4119-b659-15b8f485eb9a/30_1_Gold.pptx",
"dest": "30_1_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/30_1.pptx",
"dest": "30_1.pptx"
},
"options": {
"examine_shape": false,
"examine_modify_height": true
}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/7ae48c60-f143-4119-b659-15b8f485eb9a/30_1_Gold.pptx",
"dest": "30_1_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/7ae48c60-f143-4119-b659-15b8f485eb9a/30_1_Gold_all_fonts.pptx",
"dest": "30_1_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/30_1.pptx",
"dest": "30_1.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/30_1.pptx",
"dest": "30_1.pptx"
}
],
"options": [
{
"_note": "examine_shape=False is REQUIRED for examine_modify_height to work - see slides.py for execution order details",
"examine_modify_height": true,
"examine_shape": false
},
{
"_note": "examine_shape=False is REQUIRED for examine_modify_height to work - see slides.py for execution order details",
"examine_modify_height": true,
"examine_shape": false
}
]
},
"proxy": false
}

42
evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json
View File

@@ -1,7 +1,7 @@
{
"id": "9ec204e4-f0a3-42f8-8458-b772a6797cab",
"snapshot": "libreoffice_impress",
"instruction":"Please duplicate the last two slides and insert the copies in alternating order, so the sequence becomes: original slide A, original slide B, then duplicated slide A, duplicated slide B.",
"instruction": "Please duplicate the last two slides and insert the copies in alternating order, so the sequence becomes: original slide A, original slide B, then duplicated slide A, duplicated slide B.",
"source": "https://www.tiktok.com/@lil.d1rt_/video/7247574148887629083",
"config": [
{
@@ -58,17 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab/MLA_Workshop_061X_Works_Cited_Gold.pptx",
"dest": "MLA_Workshop_061X_Works_Cited_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/MLA_Workshop_061X_Works_Cited.pptx",
"dest": "MLA_Workshop_061X_Works_Cited.pptx"
}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab/MLA_Workshop_061X_Works_Cited_Gold.pptx",
"dest": "MLA_Workshop_061X_Works_Cited_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab/MLA_Workshop_061X_Works_Cited_Gold_all_fonts.pptx",
"dest": "MLA_Workshop_061X_Works_Cited_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/MLA_Workshop_061X_Works_Cited.pptx",
"dest": "MLA_Workshop_061X_Works_Cited.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/MLA_Workshop_061X_Works_Cited.pptx",
"dest": "MLA_Workshop_061X_Works_Cited.pptx"
}
]
},
"proxy": false
}

41
evaluation_examples/examples/libreoffice_impress/a53f80cd-4a90-4490-8310-097b011433f6.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/a53f80cd-4a90-4490-8310-097b011433f6/21_0_Gold.pptx",
"dest": "21_0_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/21_0.pptx",
"dest": "21_0.pptx"
},
"options": {}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/a53f80cd-4a90-4490-8310-097b011433f6/21_0_Gold.pptx",
"dest": "21_0_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/a53f80cd-4a90-4490-8310-097b011433f6/21_0_Gold_all_fonts_3.pptx",
"dest": "21_0_Gold_all_fonts_3.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/21_0.pptx",
"dest": "21_0.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/21_0.pptx",
"dest": "21_0.pptx"
}
]
},
"proxy": false
}

4
evaluation_examples/examples/libreoffice_impress/b8adbc24-cef2-4b15-99d5-ecbe7ff445eb.json
View File

@@ -72,7 +72,9 @@
"options": {
"examine_alignment": false,
"examine_font_name": false,
"examine_shape": false
"examine_shape": false,
"examine_font_size": false
}
},
"proxy": false

2
evaluation_examples/examples/libreoffice_impress/bf4e9888-f10f-47af-8dba-76413038b73c.json
View File

@@ -1,7 +1,7 @@
{
"id": "bf4e9888-f10f-47af-8dba-76413038b73c",
"snapshot": "libreoffice_impress",
"instruction": "I have a series of .png images named pic1.png to pic6.png saved on the Desktop. I want to create a new presentation with six blank slides, and insert one image per slide in order—pic1.png on the first slide, pic2.png on the second, and so on—following the numerical order. The goal is to prepare a presentation suitable for continuous playback in a kiosk or multimedia show. Could you help me with that?",
"instruction": "I have a series of .png images named pic1.png to pic6.png saved on the Desktop. I want to create a new presentation with six blank slides(which means no textbox or any other stuff on the page), and insert one image per slide in order—pic1.png on the first slide, pic2.png on the second, and so on—following the numerical order. The goal is to prepare a presentation suitable for continuous playback in a kiosk or multimedia show. Could you help me with that?",
"source": "https://help.libreoffice.org/6.4/en-US/text/simpress/guide/photo_album.html?DbPAR=IMPRESS",
"config": [
{

41
evaluation_examples/examples/libreoffice_impress/e4ef0baf-4b52-4590-a47e-d4d464cca2d7.json
View File

@@ -58,18 +58,35 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/e4ef0baf-4b52-4590-a47e-d4d464cca2d7/42_2_Gold.pptx",
"dest": "42_2_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/42_2.pptx",
"dest": "42_2.pptx"
},
"options": {}
"func": [
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/e4ef0baf-4b52-4590-a47e-d4d464cca2d7/42_2_Gold.pptx",
"dest": "42_2_Gold.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/e4ef0baf-4b52-4590-a47e-d4d464cca2d7/42_2_Gold_all_fonts.pptx",
"dest": "42_2_Gold_all_fonts.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/42_2.pptx",
"dest": "42_2.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/42_2.pptx",
"dest": "42_2.pptx"
}
]
},
"proxy": false
}

57
evaluation_examples/examples/libreoffice_impress/f23acfd2-c485-4b7c-a1e7-d4303ddfe864.json
View File

@@ -58,18 +58,57 @@
}
}
],
"func": "compare_pptx_files",
"expected": {
"func": [
"compare_pptx_files",
"compare_pptx_files",
"compare_pptx_files",
"compare_pptx_files"
],
"conj": "or",
"expected": [
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/f23acfd2-c485-4b7c-a1e7-d4303ddfe864/69_4_Gold.pptx",
"dest": "69_4_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/69_4.pptx",
"dest": "69_4.pptx"
},
"options": {}
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/f23acfd2-c485-4b7c-a1e7-d4303ddfe864/69_4_Gold2.pptx",
"dest": "69_4_Gold2.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/f23acfd2-c485-4b7c-a1e7-d4303ddfe864/69_4_Gold_all_fonts_1.pptx",
"dest": "69_4_Gold_all_fonts_1.pptx"
},
{
"type": "cloud_file",
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_impress/f23acfd2-c485-4b7c-a1e7-d4303ddfe864/69_4_Gold_all_fonts_2.pptx",
"dest": "69_4_Gold_all_fonts_2.pptx"
}
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/69_4.pptx",
"dest": "69_4.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/69_4.pptx",
"dest": "69_4.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/69_4.pptx",
"dest": "69_4.pptx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/69_4.pptx",
"dest": "69_4.pptx"
}
]
},
"proxy": false
}

57
evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json
View File

@@ -44,19 +44,52 @@
"os"
],
"evaluator": {
"func": "compare_image_text",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/ls.png",
"dest": "ls.png"
},
"expected": {
"type": "rule",
"rules": {
"type": "text",
"text": "ls"
"func": [
"compare_image_text",
"compare_image_text",
"compare_image_text"
],
"conj": "or",
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/ls.png",
"dest": "ls.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/ls.png",
"dest": "ls.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/ls.png",
"dest": "ls.png"
}
}
],
"expected": [
{
"type": "rule",
"rules": {
"type": "text",
"text": "ls"
}
},
{
"type": "rule",
"rules": {
"type": "text",
"text": "1s"
}
},
{
"type": "rule",
"rules": {
"type": "text",
"text": "1s"
}
}
]
},
"proxy": false
}

93
evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json
View File

@@ -69,19 +69,102 @@
"rules": {
"expected": {
"ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": [
"Kilimanjaro"
"Kili",
"kili",
"Kili.jpg",
"kili.jpg",
"Kilimanjaro",
"kilimanjaro",
"Kilimanjaro.jpg",
"kilimanjaro.jpg",
"Mount Kilimanjaro",
"mount kilimanjaro",
"Mount Kilimanjaro.jpg",
"mount kilimanjaro.jpg",
"Kilimanjaro Mountain",
"kilimanjaro mountain",
"Kilimanjaro Mountain.jpg",
"kilimanjaro mountain.jpg"
],
"6ed4239ecc2be3ec15ad65a78c5c823b9004d640b8cc83a6a7af5930f354de91": [
"Himalayas",
"Everest",
"Sagarmatha"
"everest",
"Everest.jpg",
"everest.jpg",
"Mount Everest",
"mount everest",
"Mount Everest.jpg",
"mount everest.jpg",
"Everest Mountain",
"everest mountain",
"Everest Mountain.jpg",
"everest mountain.jpg",
"Sagarmatha",
"sagarmatha",
"Sagarmatha.jpg",
"sagarmatha.jpg",
"Sagarmatha Mountain",
"sagarmatha mountain",
"Sagarmatha Mountain.jpg",
"sagarmatha mountain.jpg",
"Chomolungma",
"chomolungma",
"Chomolungma.jpg",
"chomolungma.jpg",
"Qomolangma",
"qomolangma",
"Qomolangma.jpg",
"qomolangma.jpg",
"Himalayas",
"himalayas",
"Himalayas.jpg",
"himalayas.jpg",
"Himalayas Mountain",
"himalayas mountain",
"Himalayas Mountain.jpg",
"himalayas mountain.jpg",
"Himalaya",
"himalaya",
"Himalaya.jpg",
"himalaya.jpg",
"Himalaya Mountain",
"himalaya mountain",
"Himalaya Mountain.jpg",
"himalaya mountain.jpg"
],
"79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": [
"Hua",
"hua"
"hua",
"Hua.jpg",
"hua.jpg",
"Mount Hua",
"mount hua",
"Mount Hua.jpg",
"mount hua.jpg",
"Hua Mountain",
"hua mountain",
"Hua Mountain.jpg",
"hua mountain.jpg",
"Huashan",
"huashan",
"Huashan.jpg",
"huashan.jpg",
"Hua Shan",
"hua shan",
"Hua Shan.jpg",
"hua shan.jpg",
"Huashan Mountain",
"huashan mountain",
"Huashan Mountain.jpg",
"huashan mountain.jpg",
"Hua Shan Mountain",
"hua shan mountain",
"Hua Shan Mountain.jpg",
"hua shan mountain.jpg"
]
},
"expect_in_result": true
"expect_in_result": true,
"result_not_list": true
}
}
},

2
evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json
View File

@@ -1,7 +1,7 @@
{
"id": "da922383-bfa4-4cd3-bbad-6bebab3d7742",
"snapshot": "multiapps",
"instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs opening now in pdf format and save them in their tile to /home/user/Documents/Blog.",
"instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs opening now in pdf format and save them in their title to /home/user/Documents/Blog.",
"source": "authors",
"config": [
{

7
evaluation_examples/examples/multi_apps/e135df7c-7687-4ac0-a5f0-76b74438b53e.json
View File

@@ -89,7 +89,14 @@
"path": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/multi_apps/e135df7c-7687-4ac0-a5f0-76b74438b53e/annual-enterprise-survey-2021-financial-year-provisional.html",
"dest": "annual-enterprise-survey-2021-financial-year-provisional_gold.html"
}
],
"options": [
{},
{
"ignore_sdnum": true
}
]
},
"proxy": true
}

4
evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json
View File

@@ -36,8 +36,8 @@
],
"evaluator": {
"func": [
"check_structure_sim",
"check_structure_sim"
"check_structure_sim_resized",
"check_structure_sim_resized"
],
"result": [
{

6
evaluation_examples/examples/multi_apps/f8cfa149-d1c1-4215-8dac-4a0932bad3c2.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/multi_apps/f8cfa149-d1c1-4215-8dac-4a0932bad3c2/file.xls",
"path": "/home/user/cell_search.xlsx"
"path": "/home/user/cell_search.xls"
}
]
}
@@ -47,7 +47,7 @@
{
"type": "open",
"parameters": {
"path": "/home/user/cell_search.xlsx"
"path": "/home/user/cell_search.xls"
}
}
],
@@ -65,7 +65,7 @@
"type": "rule",
"rules": {
"expect": {
"pattern": "www\\.google\\.com.*?/search\\?q=Nereida&"
"pattern": "https?://(www\\.?)?google\\.com/search\\?q=nereida(&|$)"
}
}
}

4
evaluation_examples/settings/proxy/dataimpulse.json
View File

@@ -2,8 +2,8 @@
{
"host": "gw.dataimpulse.com",
"port": 823,
"username": "fba5ac061fe18be70c6c",
"password": "e225c50bf56bdd6c",
"username": "your_username",
"password": "your_password",
"protocol": "http",
"provider": "dataimpulse",
"type": "residential",