From 5160619783573cdcfbccbe8b9e7b9c4c255bfba4 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 12 Jan 2024 12:19:23 +0800 Subject: [PATCH 1/6] ver Jan12th quickly fixed two thunderbird examples --- desktop_env/evaluators/metrics/thunderbird.py | 2 +- .../2ad9387a-65d8-4e33-ad5b-7580065a27ca.json | 72 +++++++++++++++++ .../35253b65-1c19-4304-8aa4-6884b8218fc0.json | 77 +++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json create mode 100644 evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py index b202d94..ce7d00d 100644 --- a/desktop_env/evaluators/metrics/thunderbird.py +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -85,7 +85,7 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any _value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\") #_condition_pattern: Pattern[str] = re.compile(r'(?PAND|OR) \((?P[\w ]+),(?P[\w ' + '\'' + r']+),(?:"(?P(?:[^"]|\")+)"|(?P[^)]+))\)') -_condition_pattern: Pattern[str] = re.compile(r'(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)') +_condition_pattern: Pattern[str] = re.compile(r'\b(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)|\bALL\b') def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ Args: diff --git a/evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json b/evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json new file mode 100644 index 0000000..6b89f31 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json @@ -0,0 +1,72 @@ +{ + "id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca", + "snapshot": "thunderbird", + "instruction": "Create two local folders for me: COMPANY and UNIVERSITY.", + "source": "https://support.mozilla.org/bm/questions/1027435", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/2ad9387a-65d8-4e33-ad5b-7580065a27ca", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": ["ls", "-R", "/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders"], + "stdout": "thunder-local-folder.ls" + } + } + ], + "func": "check_list", + "result": { + "type": "cache_file", + "path": "thunder-local-folder.ls" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + "\\bCOMPANY\\.msf\\b", + "\\bCOMPANY/?(?!\\.msf)", + "\\bUNIVERSITY\\.msf\\b", + "\\bUNIVERSITY/?(?!\\.msf)" + ] + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json b/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json new file mode 100644 index 0000000..a1c3154 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json @@ -0,0 +1,77 @@ +{ + "id": "35253b65-1c19-4304-8aa4-6884b8218fc0", + "snapshot": "thunderbird", + "instruction": "Set up to forward every email received by anonym-x2024@outlook.com in the future to anonym-x2024@gmail.com. Please don't touch the online account. Just locally in the Thunderbird!", + "source": "https://support.mozilla.org/en-US/questions/1259354", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/35253b65-1c19-4304-8aa4-6884b8218fc0", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "command", + "parameters": { + "command": ["wmctrl", "-Fc", "Message Filters"], + "until": { + "returncode": 1 + } + } + } + ], + "func": "check_thunderbird_filter", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat", + "dest": "msgFilterRules.dat" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + { + "enabled": "yes", + "action": "Forward", + "actionValue": "anonym-x2024@gmail.com", + "condition": ["ALL"] + } + ] + } + } + } +} From e08df5712912379f41d4f51f9eb996addadbdc49 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 12 Jan 2024 23:07:00 +0800 Subject: [PATCH 2/6] ver Jan12thv2 sqlite3 metric --- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/general.py | 7 ++ .../3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json | 74 +++++++++++++++++++ ...7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup | 52 +++++++++++++ 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json create mode 100644 evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 28a53ae..3148d2b 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -9,5 +9,5 @@ from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom from .table import compare_table from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ compare_videos -from .general import check_csv, check_accessibility_tree, check_list +from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index d0795a6..e1e2c95 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -13,6 +13,8 @@ from rapidfuzz import fuzz from .utils import _match_record +import sqlite3 + def exact_match(result, rules) -> float: expect = rules["expected"] print(result, expect) @@ -146,3 +148,8 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: # def check_existence(result: str, *args) -> float: # return 1. - (result is None) + +def run_sqlite3(result: str, rules: Dict[str, Any]) -> float: + connection = sqlite3.connect(result) + connection.execute(rules["sql"]) + return float(connection.fetchone()[0] or 0) diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json new file mode 100644 index 0000000..4f688d6 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json @@ -0,0 +1,74 @@ +{ + "id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "snapshot": "thunderbird", + "instruction": "Add a star to every email in local Bills folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "wmctrl", + "-xFc", + "Mail.thunderbird" + ], + "until": { + "returncode": 1 + } + } + } + ], + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite", + "dest": "global-messages-db.sqlite" + }, + "func": "run_sqlite3", + "expected": { + "type": "rule", + "rules": { + "sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);" + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup new file mode 100644 index 0000000..7c1e728 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup @@ -0,0 +1,52 @@ +{ + "id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "snapshot": "thunderbird", + "instruction": "Add a star to every email in local Bills folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + } + ], + "trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "wmctrl", + "-xFc", + "Mail.thunderbird" + ], + "until": { + "returncode": 1 + } + } + } + ], + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite", + "dest": "global-messages-db.sqlite" + }, + "func": "run_sqlite3", + "expected": { + "type": "rule", + "rules": { + "sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);" + } + } + } +} From d4192d3d9c24032c2d673732ce47ac28ff994761 Mon Sep 17 00:00:00 2001 From: David Chang Date: Sat, 13 Jan 2024 00:06:11 +0800 Subject: [PATCH 3/6] ver Jan12thv3 debugged --- desktop_env/evaluators/metrics/general.py | 6 +++--- main.py | 4 ++-- requirements.txt | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index e1e2c95..5b5492e 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -150,6 +150,6 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: # return 1. - (result is None) def run_sqlite3(result: str, rules: Dict[str, Any]) -> float: - connection = sqlite3.connect(result) - connection.execute(rules["sql"]) - return float(connection.fetchone()[0] or 0) + connection: sqlite3.Connection = sqlite3.connect(result) + cursor: sqlite3.Cursor = connection.execute(rules["sql"]) + return float(cursor.fetchone()[0] or 0) diff --git a/main.py b/main.py index 5da8091..6f5650a 100644 --- a/main.py +++ b/main.py @@ -44,9 +44,9 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup", "r") as f: + with open("evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 18" + example["snapshot"] = "Snapshot 20" env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" , action_space="computer_13" diff --git a/requirements.txt b/requirements.txt index 558098b..588ec1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ pyacoustid opencv-python ImageHash scikit-image +librosa From a841f27a9de503874ed8fc1a7b5a5668c7adbd1d Mon Sep 17 00:00:00 2001 From: tsuky_chen <3107760494@qq.com> Date: Sat, 13 Jan 2024 01:45:46 +0800 Subject: [PATCH 4/6] update gimp examples eval --- .../554785e9-4523-4e7a-b8e1-8016f565f56a.json | 16 +++---- .../gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 | 34 +++++++++++++++ .../7a4deb26-d57d-4ea9-9a73-630f66a7b568.json | 14 +++---- .../gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce | 42 +++++++++++++++++++ 4 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 create mode 100644 evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce diff --git a/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json b/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json index 0e4f7f8..b315b92 100644 --- a/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json +++ b/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "", - "path": "Desktop/2.png" + "url": "https://drive.usercontent.google.com/download?id=1-3cc3qqjlJ_3d9IpwsuvSuYpyJ2LxrIA&export=download&authuser=0&confirm=t&uuid=6b08e902-ed25-47d1-97d9-7794f9508288&at=APZUnTW74TY2hHtBbWDa0N7-cE58:1705060487948", + "path": "Desktop/woman_sitting_by_the_tree2.png" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/2.png" + "path": "Desktop/woman_sitting_by_the_tree2.png" } } ], @@ -29,14 +29,14 @@ "evaluator": { "func": "increase_saturation", "expected": { - "type": "cloud_file", - "path": "", - "dest": "edited_colorful.png" + "type": "vm_file", + "path": "Desktop/woman_sitting_by_the_tree2.png", + "dest": "woman_sitting_by_the_tree2.png" }, "result": { "type": "vm_file", - "path": "Desktop/2.png", - "dest": "2.png" + "path": "Desktop/edited_colorful.png", + "dest": "edited_colorful.png" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 b/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 new file mode 100644 index 0000000..b5dfebb --- /dev/null +++ b/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 @@ -0,0 +1,34 @@ +{ + "id": "77b8ab4d-994f-43ac-8930-8ca087d7c4b4", + "snapshot": "gimp", + "instruction": "Could you help me keep my photo on the desktop and rename it export.jpg?", + "source": "https://superuser.com/questions/1636113/how-to-get-gimp-to-recognize-images-or-pictures-folder-as-the-default-folder-for", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=13ECFsPxznuoCANto21ijj9OzP0APukIH&export=download&authuser=0&confirm=t&uuid=d8f2dd03-8992-4646-be62-3a3cf89583f2&at=APZUnTVsR0xmbXvpFIpXLzCcLrMa:1705062951627", + "path": "Desktop/The_Lost_River_Of_Dreams.jpg" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/The_Lost_River_Of_Dreams.jpg" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "check_file_exists", + "file_name": "export.png", + "directory": "/home/user/Desktop/" + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json b/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json index a2acd0b..ec698ff 100644 --- a/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json +++ b/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1SIvX9Wimyw6i2UvnoLTNDHIObvDLAsIM&export=download&authuser=0&confirm=t&uuid=a48447ab-13a2-421f-9662-6ffff8f6f6d5&at=APZUnTVRxofs822XxgEv33WwYCkb:1705046264363", - "path": "Desktop/1.png" + "path": "Desktop/woman_sitting_by_the_tree.png" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/1.png" + "path": "Desktop/woman_sitting_by_the_tree.png" } } ], @@ -29,14 +29,14 @@ "evaluator": { "func": "decrease_brightness", "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1M7T5iPv0kWvmZBbdTcRdG45zIpHMtrBn&export=download&authuser=0&confirm=t&uuid=d95db1f8-27dc-4532-9976-ea99c293f53e&at=APZUnTXfpgNCtXDtlqd55LgmrIAj:1705048270986", - "dest": "edited_darker.png" + "type": "vm_file", + "path": "Desktop/woman_sitting_by_the_tree.png", + "dest": "woman_sitting_by_the_tree.png" }, "result": { "type": "vm_file", - "path": "Desktop/1.png", - "dest": "1.png" + "path": "Desktop/edited_darker.png", + "dest": "edited_darker.png" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce b/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce new file mode 100644 index 0000000..3f7d42f --- /dev/null +++ b/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce @@ -0,0 +1,42 @@ +{ + "id": "f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce", + "snapshot": "gimp", + "instruction": "Help me select the yellow triangle and move it to the center of my photo.", + "source": "https://superuser.com/questions/612338/how-do-i-select-and-move-an-object-in-gimp", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1IMFTLy3lLSP8ee5b-940JjaoOd5-oGwn&export=download&authuser=0&confirm=t&uuid=0e06dffa-b602-42c6-8841-cf6eb3524e2b&at=APZUnTVpN4xHqAIAYmHZgk-UXvY4:1705081430356", + "path": "Desktop/Triangle_On_The_Side.png" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Triangle_On_The_Side.png" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "compare_triangle_positions", + "expected": { + "type": "vm_file", + "path": "Desktop/Triangle_On_The_Side.png", + "dest": "Triangle_On_The_Side.png" + }, + "result": { + "type": "vm_file", + "path": "Desktop/Triangle_In_The_Middle.png", + "dest": "Triangle_In_The_Middle.png" + } + } +} \ No newline at end of file From 136b52c8763f9f9beb2f85acf07224a5592634f8 Mon Sep 17 00:00:00 2001 From: tsuky_chen <3107760494@qq.com> Date: Sat, 13 Jan 2024 01:49:46 +0800 Subject: [PATCH 5/6] eval gimp compare pics --- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/gimp.py | 47 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 289428c..85f09ea 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -11,7 +11,7 @@ from .table import compare_table from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ compare_videos -from .gimp import increase_saturation, decrease_brightness +from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions from .general import check_csv, check_accessibility_tree, check_list from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index cac7635..1e919ef 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -20,6 +20,10 @@ def get_gimp_export_path(): # Handle the case where the configuration file is not found print("GIMP configuration file not found") return False + +def check_file_exists(directory, filename): + file_path = os.path.join(directory, filename) + return 1 if os.path.isfile(file_path) else 0 def increase_saturation(image1_path: str, image2_path: str) -> float: def calculate_saturation(image): @@ -61,6 +65,49 @@ def decrease_brightness(image1_path: str, image2_path: str) -> float: brightness2 = calculate_brightness(image2) return 1 if brightness1 > brightness2 else 0 +import cv2 +import numpy as np + +def find_yellow_triangle(image): + # Convert the image to RGBA + rgba = cv2.cvtColor(image, cv2.COLOR_BGR2RGBA) + + # define range of yellow color in HSV + lower_yellow = np.array([0, 0, 0], dtype=np.uint8) + upper_yellow = np.array([255, 255, 255], dtype=np.uint8) + + # expand the dimensions of lower and upper yellow to match the image dimensions + lower_yellow = np.reshape(lower_yellow, (1, 1, 3)) + upper_yellow = np.reshape(upper_yellow, (1, 1, 3)) + # build a mask for the yellow color + mask = cv2.inRange(rgba, lower_yellow, upper_yellow) + + # search for contours in the mask + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + # choose the largest contour + max_contour = max(contours, key=cv2.contourArea) + + # calculate the center of the contour + M = cv2.moments(max_contour) + cx = int(M['m10'] / M['m00']) + cy = int(M['m01'] / M['m00']) + + return cx, cy + +def compare_triangle_positions(image1, image2): + image1 = cv2.imread(image1, cv2.IMREAD_COLOR) + image2 = cv2.imread(image2, cv2.IMREAD_COLOR) + # find the center of the yellow triangle in each image + cx1, cy1 = find_yellow_triangle(image1) + cx2, cy2 = find_yellow_triangle(image2) + + # calculate the distance between the center of the triangle and the center of the image + center_distance1 = np.sqrt((cx1 - image1.shape[1] // 2)**2 + (cy1 - image1.shape[0] // 2)**2) + center_distance2 = np.sqrt((cx2 - image2.shape[1] // 2)**2 + (cy2 - image2.shape[0] // 2)**2) + + return 1 if center_distance1 > center_distance2 else 0 + if __name__ == "__main__": image1_path = "../Downloads/1.png" From 105fd35683d22fd4f5d29a39f7db44262e824a55 Mon Sep 17 00:00:00 2001 From: Siheng Zhao <1730625285@qq.com> Date: Sat, 13 Jan 2024 17:53:13 +0800 Subject: [PATCH 6/6] implement action replay for vscode and gimp evaluation --- desktop_env/evaluators/getters/__init__.py | 2 ++ desktop_env/evaluators/getters/replay.py | 19 +++++++++++ desktop_env/evaluators/getters/vscode.py | 13 ++++++++ desktop_env/evaluators/metrics/vscode.py | 32 ++++++++++++++++--- .../53ad5833-3455-407b-bbc6-45b4c79ab8fb.json | 18 ++++++++++- 5 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 desktop_env/evaluators/getters/vscode.py diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 7f59d4b..546b8db 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -3,3 +3,5 @@ from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper from .misc import get_rule, get_accessibility_tree from .vlc import get_vlc_playing_info, get_vlc_config from .chrome import get_default_search_engine, get_bookmarks +from .replay import get_replay +from .vscode import get_vscode_config diff --git a/desktop_env/evaluators/getters/replay.py b/desktop_env/evaluators/getters/replay.py index e69de29..a0d1d62 100644 --- a/desktop_env/evaluators/getters/replay.py +++ b/desktop_env/evaluators/getters/replay.py @@ -0,0 +1,19 @@ +from typing import List, Dict, Any + +def get_replay(env, trajectory: List[Dict[str, Any]]) -> None: + + def parse(action): + if action["type"] == "hotkey": + keys = "', '".join(action["param"]) + return f"pyautogui.hotkey('{keys}')" + + if action["type"] == "typewrite": + text = action["param"] + return f"pyautogui.typewrite('{text}')" + + if action["type"] == "press": + key = action["param"] + return f"pyautogui.press('{key}')" + + for action in trajectory: + env.controller.execute_python_command(parse(action)) \ No newline at end of file diff --git a/desktop_env/evaluators/getters/vscode.py b/desktop_env/evaluators/getters/vscode.py new file mode 100644 index 0000000..7758d17 --- /dev/null +++ b/desktop_env/evaluators/getters/vscode.py @@ -0,0 +1,13 @@ +from typing import Dict +from typing import Any +from replay import get_replay +from file import get_vm_file + +def get_vscode_config(env, config: Dict[str, Any]) -> str: + + get_replay(env, config["trajectory"]) + + return get_vm_file(env, { + "path": config["path"], + "dest": config["dest"] + }) \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index 9efef3e..a76910b 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -1,11 +1,8 @@ def compare_text_file(actual: str, expected: str, **options) -> float: """ Args: - actual (str): path to result xlsx - expected (str): path to gold xlsx - options (Dict[str, List[str]]): dict like - { - } + actual (str): path to result text file + expected (str): path to gold text file Return: float: the score @@ -20,7 +17,32 @@ def compare_text_file(actual: str, expected: str, **options) -> float: return 1.0 return 0.0 +def compare_config(actual: str, expected: str, **options) -> float: + """ + Args: + actual (str): path to result text file + expected (str): gold string + + Return: + float: the score + """ + + with open(actual) as f1: + actual_text = f1.read() + + if actual_text == expected: + return 1.0 + return 0.0 + def compare_answer(actual: str, expected: str, **options) -> float: + """ + Args: + actual (str): result string + expected (str): gold string + + Return: + float: the score + """ if actual == expected: return 1.0 diff --git a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json index 893eada..824f604 100644 --- a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json +++ b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json @@ -15,5 +15,21 @@ "related_apps": [ "vscode" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "compare_config", + "expected": { + "type": "string", + "string": "project" + }, + "result": { + "type": "vscode_config", + "trajectory": [ + {"type": "hotkey", "param": ["command", "shift", "p"]}, + {"type": "typewrite", "param": "OpenProject"}, + {"type": "press", "param": "enter"} + ], + "path": "OpenProject.txt", + "dest": "OpenProject.txt" + } + } }