From 284d6fb3791c83de07bf1704f027fb0d261f948d Mon Sep 17 00:00:00 2001 From: BlankCheng <913501223@qq.com> Date: Mon, 29 Jan 2024 21:42:16 +0800 Subject: [PATCH] Add human operation time log --- desktop_env/controllers/python.py | 3 +- desktop_env/envs/desktop_env.py | 3 +- desktop_env/server/main.py | 2 + .../06ca5602-62ca-47f6-ad4f-da151cde54cc.json | 95 +++++++++++++ .../2a729ded-3296-423d-aec4-7dd55ed5fbb3.json | 50 +++++++ .../554785e9-4523-4e7a-b8e1-8016f565f56a.json | 50 +++++++ .../72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json | 95 +++++++++++++ .../734d6579-c07d-47a8-9ae2-13339795476b.json | 99 +++++++++++++ .../7767eef2-56a3-4cea-8c9f-48c070c7d65b.json | 54 +++++++ .../77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json | 4 +- .../7a4deb26-d57d-4ea9-9a73-630f66a7b568.json | 58 +++++++- .../7b7617bd-57cc-468e-9c91-40c4ec2bcb3d.json | 18 +++ .../a746add2-cab0-4740-ac36-c3769d9bfb46.json | 32 ++++- .../b148e375-fe0b-4bec-90e7-38632b0d73c2.json | 34 ++++- .../d16c99dc-2a1e-46f2-b350-d97c86c85c15.json | 99 +++++++++++++ .../d52d6308-ec58-42b7-a2c9-de80e4837b2b.json | 18 +++ .../e2dd0213-26db-4349-abe5-d5667bfd725c.json | 90 ++++++++++++ .../f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json | 65 +++++++-- .../f723c744-e62c-4ae6-98d1-750d3cd7d79d.json | 134 ++++++++++++------ main.py | 31 ++-- 20 files changed, 958 insertions(+), 76 deletions(-) create mode 100644 evaluation_examples/examples/gimp/06ca5602-62ca-47f6-ad4f-da151cde54cc.json create mode 100644 evaluation_examples/examples/gimp/72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json create mode 100644 evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json create mode 100644 evaluation_examples/examples/gimp/7767eef2-56a3-4cea-8c9f-48c070c7d65b.json create mode 100644 evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json create mode 100644 evaluation_examples/examples/gimp/e2dd0213-26db-4349-abe5-d5667bfd725c.json diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index a4b2abb..9d0d24f 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -63,7 +63,8 @@ class PythonController: Executes a python command on the server. It can be used to execute the pyautogui commands, or... any other python command. who knows? """ - command_list = ["python", "-c", self.pkgs_prefix.format(command=command)] + # command_list = ["python", "-c", self.pkgs_prefix.format(command=command)] + command_list = ["python3", "-c", self.pkgs_prefix.format(command=command)] payload = json.dumps({"command": command_list, "shell": False}) headers = { 'Content-Type': 'application/json' diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index ef7a1ad..b074cff 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -285,7 +285,8 @@ class DesktopEnv(gym.Env): logger.info("Emulator started.") logger.info("Get meta info of the VM...") - self.vm_platform = self.controller.get_vm_platform() + # self.vm_platform = self.controller.get_vm_platform() + self.vm_platform = "Darwin" self.vm_screen_size = self.controller.get_vm_screen_size() print(self.vm_screen_size) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 357664c..e8b2b48 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -600,6 +600,8 @@ def download_file(): data = request.json url = data.get('url', None) path = data.get('path', None) + print(url, path) + print("*" * 100) if not url or not path: return "Path or URL not supplied!", 400 diff --git a/evaluation_examples/examples/gimp/06ca5602-62ca-47f6-ad4f-da151cde54cc.json b/evaluation_examples/examples/gimp/06ca5602-62ca-47f6-ad4f-da151cde54cc.json new file mode 100644 index 0000000..a903623 --- /dev/null +++ b/evaluation_examples/examples/gimp/06ca5602-62ca-47f6-ad4f-da151cde54cc.json @@ -0,0 +1,95 @@ +{ + "id": "06ca5602-62ca-47f6-ad4f-da151cde54cc", + "snapshot": "gimp", + "instruction": "Could you help me set the image to Palette-Based?", + "source": "https://stackoverflow.com/questions/74664666/how-to-export-palette-based-png-in-gimp", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1fxR-hAaF5BoONWxmzJTrIxXFw9EHTg_A", + "path": "~/Desktop/computer.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "~/Desktop/computer.png" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"palette_computer\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_palette_and_structure_sim", + "expected": { + "type": "vm_file", + "path": "~/Desktop/computer.png", + "dest": "computer.png" + }, + "result": { + "type": "vm_file", + "path": "~/Desktop/palette_computer.png", + "dest": "palette_computer.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json b/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json index 4889b3c..1e42323 100644 --- a/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json +++ b/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json @@ -34,6 +34,56 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"dog_without_background\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], "func": "check_structure_sim", "expected": { "type": "vm_file", diff --git a/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json b/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json index 8e98a3d..fa2fbd5 100644 --- a/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json +++ b/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json @@ -30,6 +30,56 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"edited_colorful\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], "func": "check_saturation_increase_and_structure_sim", "expected": { "type": "vm_file", diff --git a/evaluation_examples/examples/gimp/72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json b/evaluation_examples/examples/gimp/72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json new file mode 100644 index 0000000..df81d34 --- /dev/null +++ b/evaluation_examples/examples/gimp/72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json @@ -0,0 +1,95 @@ +{ + "id": "72f83cdc-bf76-4531-9a1b-eb893a13f8aa", + "snapshot": "gimp", + "instruction": "Please rotate my figure to mirror it horizontally.", + "source": "https://www.youtube.com/watch?v=V3slJcft6Tw", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1bS4xGh8bhQVSuRCwFscYZO_xzymAJgM8", + "path": "~/Desktop/berry.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "~/Desktop/berry.png" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"berry_mirror\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_image_mirror", + "expected": { + "type": "vm_file", + "path": "~/Desktop/berry.png", + "dest": "berry.png" + }, + "result": { + "type": "vm_file", + "path": "~/Desktop/berry_mirror.png", + "dest": "berry_mirror.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json b/evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json new file mode 100644 index 0000000..a264242 --- /dev/null +++ b/evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json @@ -0,0 +1,99 @@ +{ + "id": "734d6579-c07d-47a8-9ae2-13339795476b", + "snapshot": "gimp", + "instruction": "Could you fill the background layer with green color, leaving the object layer as is?", + "source": "https://www.youtube.com/watch?v=LX-S1CX1HUI", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=18r4e9FqlMqqD1ltiHRixZkE_Jc865wgb", + "path": "~/Desktop/white_background_with_object.xcf" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1D9rLa3XDEk5warlI0jVxcbixyr8SQm_C", + "path": "~/Desktop/white_background_with_object.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "~/Desktop/white_background_with_object.xcf" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"green_background_with_object\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_green_background", + "expected": { + "type": "vm_file", + "path": "~/Desktop/white_background_with_object.png", + "dest": "white_background_with_object.png" + }, + "result": { + "type": "vm_file", + "path": "~/Desktop/green_background_with_object.png", + "dest": "green_background_with_object.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/7767eef2-56a3-4cea-8c9f-48c070c7d65b.json b/evaluation_examples/examples/gimp/7767eef2-56a3-4cea-8c9f-48c070c7d65b.json new file mode 100644 index 0000000..a478344 --- /dev/null +++ b/evaluation_examples/examples/gimp/7767eef2-56a3-4cea-8c9f-48c070c7d65b.json @@ -0,0 +1,54 @@ +{ + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "snapshot": "gimp", + "instruction": "Please help change GIMP's theme from dark to light.", + "source": "https://www.youtube.com/watch?v=LX-S1CX1HUI", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "gimp" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "check_config_status", + "expected": { + "type": "rule", + "rules": { + "type:": "key-value", + "key": "theme", + "value": "\"Light\"" + } + }, + "result": { + "type": "gimp_config_file", + "file_name": "gimprc", + "dest": "gimprc" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json b/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json index 583ee74..7607b2e 100644 --- a/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json +++ b/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json @@ -30,7 +30,7 @@ "gimp" ], "evaluator": { - "func": "check_file_exists", + "func": "check_file_exists_and_structure_sim", "expected": { "type": "vm_file", "path": "~/Desktop/The_Lost_River_Of_Dreams.jpg", @@ -39,7 +39,7 @@ "result": { "type": "vm_file", "path": "~/Desktop/export.jpg", - "dest": "export.png" + "dest": "export.jpg" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json b/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json index 48143d3..42662fc 100644 --- a/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json +++ b/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json @@ -30,16 +30,66 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"edited_darker\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], "func": "check_brightness_decrease_and_structure_sim", "expected": { "type": "vm_file", - "path": "Desktop/woman_sitting_by_the_tree.png", - "dest": "~/woman_sitting_by_the_tree.png" + "path": "~/Desktop/woman_sitting_by_the_tree.png", + "dest": "woman_sitting_by_the_tree.png" }, "result": { "type": "vm_file", - "path": "Desktop/edited_darker.png", - "dest": "~/edited_darker.png" + "path": "~/Desktop/edited_darker.png", + "dest": "edited_darker.png" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/7b7617bd-57cc-468e-9c91-40c4ec2bcb3d.json b/evaluation_examples/examples/gimp/7b7617bd-57cc-468e-9c91-40c4ec2bcb3d.json index c55a1a4..24a66c5 100644 --- a/evaluation_examples/examples/gimp/7b7617bd-57cc-468e-9c91-40c4ec2bcb3d.json +++ b/evaluation_examples/examples/gimp/7b7617bd-57cc-468e-9c91-40c4ec2bcb3d.json @@ -18,6 +18,24 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_config_status", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/gimp/a746add2-cab0-4740-ac36-c3769d9bfb46.json b/evaluation_examples/examples/gimp/a746add2-cab0-4740-ac36-c3769d9bfb46.json index 871fdc0..cf166ae 100644 --- a/evaluation_examples/examples/gimp/a746add2-cab0-4740-ac36-c3769d9bfb46.json +++ b/evaluation_examples/examples/gimp/a746add2-cab0-4740-ac36-c3769d9bfb46.json @@ -4,11 +4,23 @@ "instruction": "Help me open up the Vignette filter window.", "source": "https://www.youtube.com/watch?v=_L_MMU22bAw", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1IQPiNT6Vmd0YO0OytaB8lb_Dvo1mCGnP", + "path": "~/Desktop/dog_with_background.png" + } + ] + } + }, { "type": "launch", "parameters": { "command": [ - "gimp" + "gimp", + "~/Desktop/dog_with_background.png" ] } } @@ -18,6 +30,24 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"esc\"]);pyautogui.hotkey([\"ctrl\", \"q\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_config_status", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/gimp/b148e375-fe0b-4bec-90e7-38632b0d73c2.json b/evaluation_examples/examples/gimp/b148e375-fe0b-4bec-90e7-38632b0d73c2.json index 0cd52d8..16be260 100644 --- a/evaluation_examples/examples/gimp/b148e375-fe0b-4bec-90e7-38632b0d73c2.json +++ b/evaluation_examples/examples/gimp/b148e375-fe0b-4bec-90e7-38632b0d73c2.json @@ -4,11 +4,23 @@ "instruction": "Could you assist me in adding a new layer and naming it 'Square'?", "source": "https://www.quora.com/How-do-I-add-layers-in-GIMP", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1CqcNw_8O0VeLuD0VkVTgDHOoUMFQNOXN", + "path": "~/Desktop/white_background.xcf" + } + ] + } + }, { "type": "launch", "parameters": { "command": [ - "gimp" + "gimp", + "~/Desktop/white_background.xcf" ] } } @@ -18,13 +30,31 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_config_status", "expected": { "type": "rule", "rules": { "type:": "key-value", "key": "layer-new-name", - "value": "Square" + "value": "\"Square\"" } }, "result": { diff --git a/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json b/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json new file mode 100644 index 0000000..b9f0525 --- /dev/null +++ b/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json @@ -0,0 +1,99 @@ +{ + "id": "d16c99dc-2a1e-46f2-b350-d97c86c85c15", + "snapshot": "gimp", + "instruction": "Could you assist me with resizing just one layer of an image? I need to adjust the height to 512 pixels while maintaining the original aspect ratio?", + "source": "https://stackoverflow.com/questions/75185543/use-gimp-to-resize-image-in-one-layer-only", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=14ZEnUNHrIVDvcyF6A6EFgma6LcgPWj1X", + "path": "~/Desktop/dog_with_background.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1wq2sjBX8X-SGIEj1GKLGAuQ8BNVYqa0-", + "path": "~/Desktop/dog_with_background_two_layers.xcf" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "~/Desktop/dog_with_background_two_layers.xcf" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"resized\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_image_size_and_structure_sim", + "expected":{ + "type": "vm_file", + "path": "~/Desktop/dog_with_background.png", + "dest": "dog_with_background.png" + }, + "result": { + "type": "vm_file", + "path": "~/Desktop/resized.png", + "dest": "resized.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/d52d6308-ec58-42b7-a2c9-de80e4837b2b.json b/evaluation_examples/examples/gimp/d52d6308-ec58-42b7-a2c9-de80e4837b2b.json index 943660f..e4ca47f 100644 --- a/evaluation_examples/examples/gimp/d52d6308-ec58-42b7-a2c9-de80e4837b2b.json +++ b/evaluation_examples/examples/gimp/d52d6308-ec58-42b7-a2c9-de80e4837b2b.json @@ -18,6 +18,24 @@ "gimp" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"ctrl\", \"q\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_config_status", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/gimp/e2dd0213-26db-4349-abe5-d5667bfd725c.json b/evaluation_examples/examples/gimp/e2dd0213-26db-4349-abe5-d5667bfd725c.json new file mode 100644 index 0000000..531cb2b --- /dev/null +++ b/evaluation_examples/examples/gimp/e2dd0213-26db-4349-abe5-d5667bfd725c.json @@ -0,0 +1,90 @@ +{ + "id": "e2dd0213-26db-4349-abe5-d5667bfd725c", + "snapshot": "gimp", + "instruction": "Can you assist me in shifting the text box to the left? I keep accidentally selecting the image layer beneath it.", + "source": "https://superuser.com/questions/839650/how-to-move-an-inserted-text-box-in-gimp", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1KMHuScgF8Eq9OPR40SCntxO3Q1DUmbKV", + "path": "~/Desktop/orange_background.xcf" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "~/Desktop/orange_background.xcf" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"leftside_textbox\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_textbox_on_leftside", + "result": { + "type": "vm_file", + "path": "~/Desktop/leftside_textbox.png", + "dest": "leftside_textbox.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json b/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json index 5d79f5d..ad5005b 100644 --- a/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json +++ b/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json @@ -9,8 +9,12 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1IMFTLy3lLSP8ee5b-940JjaoOd5-oGwn&export=download&authuser=0&confirm=t&uuid=0e06dffa-b602-42c6-8841-cf6eb3524e2b&at=APZUnTVpN4xHqAIAYmHZgk-UXvY4:1705081430356", + "url": "https://drive.google.com/uc?export=download&id=1p0m5meh7NIT4v6hNdqJqK3GCcpZiMapB", "path": "~/Desktop/Triangle_On_The_Side.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1w7f2fW2mWkUbteQvQd-lepHzvBC9GaG_", + "path": "~/Desktop/Triangle_On_The_Side.xcf" } ] } @@ -20,7 +24,7 @@ "parameters": { "command": [ "gimp", - "~/Desktop/Triangle_On_The_Side.png" + "~/Desktop/Triangle_On_The_Side.xcf" ] } } @@ -30,12 +34,57 @@ "gimp" ], "evaluator": { - "func": "compare_triangle_positions", - "expected": { - "type": "vm_file", - "path": "~/Desktop/Triangle_On_The_Side.png", - "dest": "Triangle_On_The_Side.png" - }, + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"Triangle_In_The_Middle\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_triangle_position", "result": { "type": "vm_file", "path": "~/Desktop/Triangle_In_The_Middle.png", diff --git a/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json b/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json index 9651c1e..85f21f8 100644 --- a/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json +++ b/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json @@ -1,45 +1,95 @@ { - "id": "f723c744-e62c-4ae6-98d1-750d3cd7d79d", - "snapshot": "gimp", - "instruction": "I'd like to make the picture's contrast stronger to really bring out the main subject. Could you assist me in boosting the contrast?", - "source": "https://www.reddit.com/r/GIMP/comments/12e57w8/how_to_use_gimp_to_exaggerate_contrast/", - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F", - "path": "~/Desktop/berries.png" - } - ] - } - }, - { - "type": "launch", - "parameters": { - "command": [ - "gimp", - "~/Desktop/berries.png" - ] - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "gimp" - ], - "evaluator": { - "func": "check_contrast_increase_and_structure_sim", - "expected": { - "type": "vm_file", - "path": "~/Desktop/berries.png", - "dest": "berries.png" - }, - "result": { - "type": "vm_file", - "path": "~/Desktop/berries_contrast.png", - "dest": "berries_contrast.png" - } + "id": "f723c744-e62c-4ae6-98d1-750d3cd7d79d", + "snapshot": "gimp", + "instruction": "I'd like to make the picture's contrast stronger to really bring out the main subject. Could you assist me in boosting the contrast?", + "source": "https://www.reddit.com/r/GIMP/comments/12e57w8/how_to_use_gimp_to_exaggerate_contrast/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F", + "path": "~/Desktop/berries.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "~/Desktop/berries.png" + ] + } } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.hotkey([\"shift\", \"ctrl\", \"e\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.write(\"berries_contrast\");pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "-c", + "import pyautogui; pyautogui.press([\"enter\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "check_contrast_increase_and_structure_sim", + "expected": { + "type": "vm_file", + "path": "~/Desktop/berries.png", + "dest": "berries.png" + }, + "result": { + "type": "vm_file", + "path": "~/Desktop/berries_contrast.png", + "dest": "berries_contrast.png" + } + } } \ No newline at end of file diff --git a/main.py b/main.py index c2f770e..44cd510 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ import json import logging import os import sys +import time from desktop_env.envs.desktop_env import DesktopEnv @@ -46,12 +47,11 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json", "r") as f: + with open("evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json", "r") as f: example = json.load(f) - example["snapshot"] = "base18" - + example["snapshot"] = "快照 9" env = DesktopEnv( - path_to_vm=r"D:\Ubuntu\Ubuntu\Ubuntu.vmx", + path_to_vm=r"~/Virtual Machines.localized/ubuntu_vm.vmwarevm/ubuntu_vm.vmx", action_space="computer_13", task_config=example ) @@ -60,14 +60,14 @@ def human_agent(): done = False trajectory = [ - { - "action_type": "MOVE_TO", - "parameters": { - "x": 754, - "y": 1057 - } - }, - {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}} + # { + # "action_type": "MOVE_TO", + # "parameters": { + # "x": 1, + # "y": 1 + # } + # }, + # {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}} ] for i in range(len(trajectory)): @@ -91,13 +91,14 @@ def human_agent(): logger.info("The episode is done.") break - #input("PAUSING") + input("Press Enter to start human operation...") + human_start_time = time.time() + input("Press Enter to finish human operation.") + print("Time elapsed of human operation: %.2f" % (time.time() - human_start_time)) result = env.evaluate() logger.info("Result: %.2f", result) - #input("PAUSING") - # env.close() logger.info("Environment closed.")