Merge remote-tracking branch 'origin/main'

desktop_env/evaluators/metrics/table.py

@@ -392,17 +392,13 @@ if __name__ == '__main__':
     logger.addHandler(stdout_handler)
     logger.addHandler(sdebug_handler)
 
-    path1 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number.xlsx"
+    path1 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday.xlsx"
-    path2 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number_gold.xlsx"
+    path2 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday_gold.xlsx"
-    rules = [{"type": "check_cell"
+    rules = [ { "type": "sheet_data"
-                 , "sheet_idx": 0
+              , "sheet_idx0": 0
-                 , "coordinate": "E3"
+              , "sheet_idx1": "EI0"
-                 , "props": {"value": {"method": "approx:0.001"
-            , "ref": 191.6667
-                                       }
-                             }
               }
-             ]
+            ]
     print(compare_table(path1, path2
                         , rules=rules
                         )

evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
     "func": "compare_table",

evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json

@@ -69,6 +69,12 @@
 						"/home/user/Represent_in_millions_billions.xlsx"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
     "func": "compare_table",

evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json

@@ -50,12 +50,70 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+			{
+				"type": "download",
+				"parameters": {
+					"files": [
+						{
+							"url": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
+							"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx"
+						}
+					]
+				}
+			},
+			{
+				"type": "open",
+				"parameters": {
+					"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx"
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 2
+				}
+			},
+			{
+				"type": "activate_window",
+				"parameters": {
+					"window_name": "Employee_Age_By_Birthday_gold.xlsx - LibreOffice Calc",
+					"strict": true
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python",
+						"-c",
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",
 		"expected": {
-			"type": "cloud_file",
+			"type": "vm_file",
-			"path": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
+			"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx",
 			"dest": "Employee_Age_By_Birthday_gold.xlsx"
 		},
 		"result": {

evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json

@@ -69,6 +69,12 @@
 						"/home/user/Padding_Decimals_In_Formular.xlsx"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json

@@ -69,6 +69,12 @@
 						"/home/user/Keep_Two_decimal_points.xlsx"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
     "func": "compare_table",

evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json

@@ -1,7 +1,7 @@
 {
 	"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
 	"snapshot": "libreoffice_calc",
-	"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
+	"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you help me to update all the numbers in the sheet? Also please keep the decimal numbers as-is.",
 	"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
 	"config": [
 		{
@@ -69,6 +69,12 @@
 						"/home/user/Set_Decimal_Separator_Dot.xlsx"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json

@@ -50,6 +50,12 @@
 						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
 			}
 		],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
     "func": "compare_table",

evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
 		"func": "compare_table",

evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
     "func": "compare_table",

evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json

@@ -50,6 +50,12 @@
             "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
     ],
 		"func": "compare_table",

experiment_screenshot_som.py

@@ -1,4 +1,4 @@
-import ctypes
+#import ctypes
 import datetime
 import json
 import logging
@@ -131,8 +131,10 @@ def main(example_class, example_id):
         example = json.load(f)
     example["snapshot"] = "exp_v5"
 
+    logger.info("TASK: %s/%s", example_class, example_id)
+
     api_key = os.environ.get("OPENAI_API_KEY")
-    agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'],
+    agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, max_tokens=1000, instruction=example['instruction'],
                         action_space=action_space, exp="som")
 
     # api_key = os.environ.get("GENAI_API_KEY")