Add human operation time log

desktop_env/controllers/python.py

@@ -63,7 +63,8 @@ class PythonController:
         Executes a python command on the server.
         It can be used to execute the pyautogui commands, or... any other python command. who knows?
         """
-        command_list = ["python", "-c", self.pkgs_prefix.format(command=command)]
+        # command_list = ["python", "-c", self.pkgs_prefix.format(command=command)]
+        command_list = ["python3", "-c", self.pkgs_prefix.format(command=command)]
         payload = json.dumps({"command": command_list, "shell": False})
         headers = {
             'Content-Type': 'application/json'

desktop_env/envs/desktop_env.py

@@ -285,7 +285,8 @@ class DesktopEnv(gym.Env):
         logger.info("Emulator started.")
 
         logger.info("Get meta info of the VM...")
-        self.vm_platform = self.controller.get_vm_platform()
+        # self.vm_platform = self.controller.get_vm_platform()
+        self.vm_platform = "Darwin"
         self.vm_screen_size = self.controller.get_vm_screen_size()
         print(self.vm_screen_size)
 

desktop_env/server/main.py

@@ -600,6 +600,8 @@ def download_file():
     data = request.json
     url = data.get('url', None)
     path = data.get('path', None)
+    print(url, path)
+    print("*" * 100)
 
     if not url or not path:
         return "Path or URL not supplied!", 400

evaluation_examples/examples/gimp/06ca5602-62ca-47f6-ad4f-da151cde54cc.json

@@ -0,0 +1,95 @@
+{
+  "id": "06ca5602-62ca-47f6-ad4f-da151cde54cc",
+  "snapshot": "gimp",
+  "instruction": "Could you help me set the image to Palette-Based?",
+  "source": "https://stackoverflow.com/questions/74664666/how-to-export-palette-based-png-in-gimp",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.google.com/uc?export=download&id=1fxR-hAaF5BoONWxmzJTrIxXFw9EHTg_A",
+            "path": "~/Desktop/computer.png"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "gimp",
+          "~/Desktop/computer.png"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.write(\"palette_computer\");pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 2
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 5
+        }
+      }
+    ],
+    "func": "check_palette_and_structure_sim",
+    "expected": {
+      "type": "vm_file",
+      "path": "~/Desktop/computer.png",
+      "dest": "computer.png"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "~/Desktop/palette_computer.png",
+      "dest": "palette_computer.png"
+    }
+  }
+}

evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json

@@ -34,6 +34,56 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.write(\"dog_without_background\");pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+            {
+				"type": "sleep",
+				"parameters": {
+					"seconds": 2
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 5
+				}
+			}
+		],
         "func": "check_structure_sim",
         "expected": {
             "type": "vm_file",

evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json

@@ -30,6 +30,56 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.write(\"edited_colorful\");pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+            {
+				"type": "sleep",
+				"parameters": {
+					"seconds": 2
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 5
+				}
+			}
+		],
         "func": "check_saturation_increase_and_structure_sim",
         "expected": {
             "type": "vm_file",

evaluation_examples/examples/gimp/72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json

@@ -0,0 +1,95 @@
+{
+  "id": "72f83cdc-bf76-4531-9a1b-eb893a13f8aa",
+  "snapshot": "gimp",
+  "instruction": "Please rotate my figure to mirror it horizontally.",
+  "source": "https://www.youtube.com/watch?v=V3slJcft6Tw",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.google.com/uc?export=download&id=1bS4xGh8bhQVSuRCwFscYZO_xzymAJgM8",
+            "path": "~/Desktop/berry.png"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "gimp",
+          "~/Desktop/berry.png"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.write(\"berry_mirror\");pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 2
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 5
+        }
+      }
+    ],
+    "func": "check_image_mirror",
+    "expected": {
+      "type": "vm_file",
+      "path": "~/Desktop/berry.png",
+      "dest": "berry.png"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "~/Desktop/berry_mirror.png",
+      "dest": "berry_mirror.png"
+    }
+  }
+}

evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json

@@ -0,0 +1,99 @@
+{
+  "id": "734d6579-c07d-47a8-9ae2-13339795476b",
+  "snapshot": "gimp",
+  "instruction": "Could you fill the background layer with green color, leaving the object layer as is?",
+  "source": "https://www.youtube.com/watch?v=LX-S1CX1HUI",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.google.com/uc?export=download&id=18r4e9FqlMqqD1ltiHRixZkE_Jc865wgb",
+            "path": "~/Desktop/white_background_with_object.xcf"
+          },
+          {
+            "url": "https://drive.google.com/uc?export=download&id=1D9rLa3XDEk5warlI0jVxcbixyr8SQm_C",
+            "path": "~/Desktop/white_background_with_object.png"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "gimp",
+          "~/Desktop/white_background_with_object.xcf"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.write(\"green_background_with_object\");pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 2
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 5
+        }
+      }
+    ],
+    "func": "check_green_background",
+    "expected": {
+      "type": "vm_file",
+      "path": "~/Desktop/white_background_with_object.png",
+      "dest": "white_background_with_object.png"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "~/Desktop/green_background_with_object.png",
+      "dest": "green_background_with_object.png"
+    }
+  }
+}

evaluation_examples/examples/gimp/7767eef2-56a3-4cea-8c9f-48c070c7d65b.json

@@ -0,0 +1,54 @@
+{
+  "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b",
+  "snapshot": "gimp",
+  "instruction": "Please help change GIMP's theme from dark to light.",
+  "source": "https://www.youtube.com/watch?v=LX-S1CX1HUI",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "gimp"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "check_config_status",
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "type:": "key-value",
+        "key": "theme",
+        "value": "\"Light\""
+      }
+    },
+    "result": {
+      "type": "gimp_config_file",
+      "file_name": "gimprc",
+      "dest": "gimprc"
+    }
+  }
+}

evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json

@@ -30,7 +30,7 @@
         "gimp"
     ],
     "evaluator": {
-        "func": "check_file_exists",
+        "func": "check_file_exists_and_structure_sim",
         "expected": {
             "type": "vm_file",
             "path": "~/Desktop/The_Lost_River_Of_Dreams.jpg",
@@ -39,7 +39,7 @@
         "result": {
             "type": "vm_file",
             "path": "~/Desktop/export.jpg",
-            "dest": "export.png"
+            "dest": "export.jpg"
         }
     }
 }

evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json

@@ -30,16 +30,66 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.write(\"edited_darker\");pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+            {
+				"type": "sleep",
+				"parameters": {
+					"seconds": 2
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 5
+				}
+			}
+		],
         "func": "check_brightness_decrease_and_structure_sim",
         "expected": {
             "type": "vm_file",
-            "path": "Desktop/woman_sitting_by_the_tree.png",
+            "path": "~/Desktop/woman_sitting_by_the_tree.png",
-            "dest": "~/woman_sitting_by_the_tree.png"
+            "dest": "woman_sitting_by_the_tree.png"
         },
         "result": {
             "type": "vm_file",
-            "path": "Desktop/edited_darker.png",
+            "path": "~/Desktop/edited_darker.png",
-            "dest": "~/edited_darker.png"
+            "dest": "edited_darker.png"
         }
     }
 }

evaluation_examples/examples/gimp/7b7617bd-57cc-468e-9c91-40c4ec2bcb3d.json

@@ -18,6 +18,24 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			}
+		],
         "func": "check_config_status",
         "expected": {
             "type": "rule",

evaluation_examples/examples/gimp/a746add2-cab0-4740-ac36-c3769d9bfb46.json

@@ -4,11 +4,23 @@
     "instruction": "Help me open up the Vignette filter window.",
     "source": "https://www.youtube.com/watch?v=_L_MMU22bAw",
     "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1IQPiNT6Vmd0YO0OytaB8lb_Dvo1mCGnP",
+                        "path": "~/Desktop/dog_with_background.png"
+                    }
+                ]
+            }
+        },
         {
             "type": "launch",
             "parameters": {
                 "command": [
-                    "gimp"
+                    "gimp",
+                    "~/Desktop/dog_with_background.png"
                 ]
             }
         }
@@ -18,6 +30,24 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.press([\"esc\"]);pyautogui.hotkey([\"ctrl\", \"q\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			}
+		],
         "func": "check_config_status",
         "expected": {
             "type": "rule",

evaluation_examples/examples/gimp/b148e375-fe0b-4bec-90e7-38632b0d73c2.json

@@ -4,11 +4,23 @@
     "instruction": "Could you assist me in adding a new layer and naming it 'Square'?",
     "source": "https://www.quora.com/How-do-I-add-layers-in-GIMP",
     "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1CqcNw_8O0VeLuD0VkVTgDHOoUMFQNOXN",
+                        "path": "~/Desktop/white_background.xcf"
+                    }
+                ]
+            }
+        },
         {
             "type": "launch",
             "parameters": {
                 "command": [
-                    "gimp"
+                    "gimp",
+                    "~/Desktop/white_background.xcf"
                 ]
             }
         }
@@ -18,13 +30,31 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			}
+		],
         "func": "check_config_status",
         "expected": {
             "type": "rule",
             "rules": {
                 "type:": "key-value",
                 "key": "layer-new-name",
-                "value": "Square"
+                "value": "\"Square\""
             }
         },
         "result": {

evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json

@@ -0,0 +1,99 @@
+{
+  "id": "d16c99dc-2a1e-46f2-b350-d97c86c85c15",
+  "snapshot": "gimp",
+  "instruction": "Could you assist me with resizing just one layer of an image? I need to adjust the height to 512 pixels while maintaining the original aspect ratio?",
+  "source": "https://stackoverflow.com/questions/75185543/use-gimp-to-resize-image-in-one-layer-only",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.google.com/uc?export=download&id=14ZEnUNHrIVDvcyF6A6EFgma6LcgPWj1X",
+            "path": "~/Desktop/dog_with_background.png"
+          },
+          {
+            "url": "https://drive.google.com/uc?export=download&id=1wq2sjBX8X-SGIEj1GKLGAuQ8BNVYqa0-",
+            "path": "~/Desktop/dog_with_background_two_layers.xcf"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "gimp",
+          "~/Desktop/dog_with_background_two_layers.xcf"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.write(\"resized\");pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 2
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 5
+        }
+      }
+    ],
+    "func": "check_image_size_and_structure_sim",
+    "expected":{
+        "type": "vm_file",
+        "path": "~/Desktop/dog_with_background.png",
+        "dest": "dog_with_background.png"
+      },
+    "result": {
+      "type": "vm_file",
+      "path": "~/Desktop/resized.png",
+      "dest": "resized.png"
+    }
+  }
+}

evaluation_examples/examples/gimp/d52d6308-ec58-42b7-a2c9-de80e4837b2b.json

@@ -18,6 +18,24 @@
         "gimp"
     ],
     "evaluator": {
+        "postconfig": [
+			{
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			}
+		],
         "func": "check_config_status",
         "expected": {
             "type": "rule",

evaluation_examples/examples/gimp/e2dd0213-26db-4349-abe5-d5667bfd725c.json

@@ -0,0 +1,90 @@
+{
+  "id": "e2dd0213-26db-4349-abe5-d5667bfd725c",
+  "snapshot": "gimp",
+  "instruction": "Can you assist me in shifting the text box to the left? I keep accidentally selecting the image layer beneath it.",
+  "source": "https://superuser.com/questions/839650/how-to-move-an-inserted-text-box-in-gimp",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.google.com/uc?export=download&id=1KMHuScgF8Eq9OPR40SCntxO3Q1DUmbKV",
+            "path": "~/Desktop/orange_background.xcf"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "gimp",
+          "~/Desktop/orange_background.xcf"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.write(\"leftside_textbox\");pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 2
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 5
+        }
+      }
+    ],
+    "func": "check_textbox_on_leftside",
+    "result": {
+      "type": "vm_file",
+      "path": "~/Desktop/leftside_textbox.png",
+      "dest": "leftside_textbox.png"
+    }
+  }
+}

evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json

@@ -9,8 +9,12 @@
             "parameters": {
                 "files": [
                     {
-                        "url": "https://drive.usercontent.google.com/download?id=1IMFTLy3lLSP8ee5b-940JjaoOd5-oGwn&export=download&authuser=0&confirm=t&uuid=0e06dffa-b602-42c6-8841-cf6eb3524e2b&at=APZUnTVpN4xHqAIAYmHZgk-UXvY4:1705081430356",
+                        "url": "https://drive.google.com/uc?export=download&id=1p0m5meh7NIT4v6hNdqJqK3GCcpZiMapB",
                         "path": "~/Desktop/Triangle_On_The_Side.png"
+                    },
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1w7f2fW2mWkUbteQvQd-lepHzvBC9GaG_",
+                        "path": "~/Desktop/Triangle_On_The_Side.xcf"
                     }
                 ]
             }
@@ -20,7 +24,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/Triangle_On_The_Side.png"
+                    "~/Desktop/Triangle_On_The_Side.xcf"
                 ]
             }
         }
@@ -30,12 +34,57 @@
         "gimp"
     ],
     "evaluator": {
-        "func": "compare_triangle_positions",
+        "postconfig": [
-        "expected": {
+			{
-            "type": "vm_file",
+				"type": "execute",
-            "path": "~/Desktop/Triangle_On_The_Side.png",
+				"parameters": {
-            "dest": "Triangle_On_The_Side.png"
+					"command": [
-        },
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 0.5
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.write(\"Triangle_In_The_Middle\");pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+            {
+				"type": "sleep",
+				"parameters": {
+					"seconds": 2
+				}
+			},
+            {
+				"type": "execute",
+				"parameters": {
+					"command": [
+						"python3",
+						"-c",
+						"import pyautogui; pyautogui.press([\"enter\"]);"
+					]
+				}
+			},
+			{
+				"type": "sleep",
+				"parameters": {
+					"seconds": 5
+				}
+			}
+		],
+        "func": "check_triangle_position",
         "result": {
             "type": "vm_file",
             "path": "~/Desktop/Triangle_In_The_Middle.png",

evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json

@@ -1,45 +1,95 @@
 {
-    "id": "f723c744-e62c-4ae6-98d1-750d3cd7d79d",
+  "id": "f723c744-e62c-4ae6-98d1-750d3cd7d79d",
-    "snapshot": "gimp",
+  "snapshot": "gimp",
-    "instruction": "I'd like to make the picture's contrast stronger to really bring out the main subject. Could you assist me in boosting the contrast?",
+  "instruction": "I'd like to make the picture's contrast stronger to really bring out the main subject. Could you assist me in boosting the contrast?",
-    "source": "https://www.reddit.com/r/GIMP/comments/12e57w8/how_to_use_gimp_to_exaggerate_contrast/",
+  "source": "https://www.reddit.com/r/GIMP/comments/12e57w8/how_to_use_gimp_to_exaggerate_contrast/",
-    "config": [
+  "config": [
-        {
+    {
-            "type": "download",
+      "type": "download",
-            "parameters": {
+      "parameters": {
-                "files": [
+        "files": [
-                    {
+          {
-                        "url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F",
+            "url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F",
-                        "path": "~/Desktop/berries.png"
+            "path": "~/Desktop/berries.png"
-                    }
+          }
-                ]
+        ]
-            }
+      }
-        },
+    },
-        {
+    {
-            "type": "launch",
+      "type": "launch",
-            "parameters": {
+      "parameters": {
-                "command": [
+        "command": [
-                    "gimp",
+          "gimp",
-                    "~/Desktop/berries.png"
+          "~/Desktop/berries.png"
-                ]
+        ]
-            }
+      }
-        }
-    ],
-    "trajectory": "trajectories/",
-    "related_apps": [
-        "gimp"
-    ],
-    "evaluator": {
-        "func": "check_contrast_increase_and_structure_sim",
-        "expected": {
-            "type": "vm_file",
-            "path": "~/Desktop/berries.png",
-            "dest": "berries.png"
-        },
-        "result": {
-            "type": "vm_file",
-            "path": "~/Desktop/berries_contrast.png",
-            "dest": "berries_contrast.png"
-        }
     }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "gimp"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.write(\"berries_contrast\");pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 2
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "-c",
+            "import pyautogui; pyautogui.press([\"enter\"]);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 5
+        }
+      }
+    ],
+    "func": "check_contrast_increase_and_structure_sim",
+    "expected": {
+      "type": "vm_file",
+      "path": "~/Desktop/berries.png",
+      "dest": "berries.png"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "~/Desktop/berries_contrast.png",
+      "dest": "berries_contrast.png"
+    }
+  }
 }

main.py

@@ -3,6 +3,7 @@ import json
 import logging
 import os
 import sys
+import time
 
 from desktop_env.envs.desktop_env import DesktopEnv
 
@@ -46,12 +47,11 @@ def human_agent():
     Runs the Gym environment with human input.
     """
 
-    with open("evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json", "r") as f:
+    with open("evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json", "r") as f:
         example = json.load(f)
-    example["snapshot"] = "base18"
+    example["snapshot"] = "快照 9"
-
     env = DesktopEnv(
-        path_to_vm=r"D:\Ubuntu\Ubuntu\Ubuntu.vmx",
+        path_to_vm=r"~/Virtual Machines.localized/ubuntu_vm.vmwarevm/ubuntu_vm.vmx",
         action_space="computer_13",
         task_config=example
     )
@@ -60,14 +60,14 @@ def human_agent():
     done = False
 
     trajectory = [
-        {
+        # {
-            "action_type": "MOVE_TO",
+        #     "action_type": "MOVE_TO",
-            "parameters": {
+        #     "parameters": {
-                "x": 754,
+        #         "x": 1,
-                "y": 1057
+        #         "y": 1
-            }
+        #     }
-        },
+        # },
-        {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
+        # {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
     ]
 
     for i in range(len(trajectory)):
@@ -91,13 +91,14 @@ def human_agent():
             logger.info("The episode is done.")
             break
 
-    #input("PAUSING")
+    input("Press Enter to start human operation...")
+    human_start_time = time.time()
+    input("Press Enter to finish human operation.")
+    print("Time elapsed of human operation: %.2f" % (time.time() - human_start_time))
 
     result = env.evaluate()
     logger.info("Result: %.2f", result)
 
-    #input("PAUSING")
-
     # env.close()
     logger.info("Environment closed.")