diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index 914211a..5c2dcdf 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -13,6 +13,7 @@ from odf.text import P from odf.text import Span from skimage.color import deltaE_ciede2000 from skimage.color import rgb2lab +from fuzzywuzzy import fuzz logger = logging.getLogger("desktopenv.metric.docs") @@ -57,6 +58,8 @@ def contains_page_break(docx_file): def compare_docx_files(file1, file2, **options): ignore_blanks = options.get('ignore_blanks', True) + content_only = options.get('content_only', False) + def get_paragraph_texts_odt(document): paragraphs = document.getElementsByType(P) paragraph_texts = [] @@ -89,6 +92,13 @@ def compare_docx_files(file1, file2, **options): print("Unsupported file types or mismatch between file types.") return 0 + if content_only: + # Compare the content of the documents + text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() + text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip() + similarity = fuzz.ratio(text1, text2) / 100.0 + return similarity + # Process and compare documents if ignore_blanks: text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index b641d43..058015f 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -1,5 +1,7 @@ +import copy +import json from typing import Dict -import json, copy + def check_json_keybindings(actual: str, expected: str, **options) -> float: """ @@ -10,6 +12,7 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float: Return: float: the score """ + def direct_load_json(fp): try: with open(fp, 'r') as f: @@ -17,7 +20,7 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float: return data except: return None - + def skip_first_line_load_json(fp): try: with open(fp, 'r') as f: @@ -54,7 +57,7 @@ def check_json_settings(actual: str, expected: str, **options) -> float: with open(actual, 'r') as f: data = json.load(f) - + expect = expected['expected'] data_copy = copy.deepcopy(data) data_copy.update(expect) @@ -128,3 +131,11 @@ def is_extension_installed(actual: str, rules: Dict, **options): return 0.0 else: raise NotImplementedError + + +def check_python_file_by_test_suite(actual_file, test_suites, **options) -> float: + pass + + +def check_python_file_by_gold_file(actual_file, gold_file: str, **options) -> float: + pass diff --git a/evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json b/evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json index a113665..dd6093d 100644 --- a/evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json +++ b/evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json @@ -1,9 +1,48 @@ { "id": "0c825995-5b70-4526-b663-113f4c999dd2", "snapshot": "libreoffice_calc", - "instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)'? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!", + "instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)' under /environment_policy folder? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!", "source": "authors", "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'environment_policy_report (draft).doc' or title = 'environment_policy_report (draft).docx' or title = 'environment_policy_report (draft)'", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, { "type": "command", "parameters": { @@ -30,16 +69,26 @@ "os" ], "evaluator": { - "postconfig": [], - "func": "", - "result": { - + "func": "compare_docx_files", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "path_list": [ + [ + "environment_policy_report (draft).docx" + ] + ], + "dest": [ + "environment_policy_report (draft).docx" + ] }, - "expected": { - + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1A2ti9JncAfIa6ks7FTJWHtYlZo-68FtM&export=download", + "dest": "environment_policy_report (draft)_gold.docx" }, - "options": { - + "options": { + "content_only": true } } } diff --git a/evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json b/evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json index 1317148..ce255ca 100644 --- a/evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json +++ b/evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json @@ -49,6 +49,8 @@ ], "trajectory": "trajectories/26150609-0da3-4a7d-8868-0faf9c5f01bb", "related_apps": [ + "vs_code", + "os" ], "evaluator": { "postconfig": [], diff --git a/evaluation_examples/examples/multi_apps/881deb30-9549-4583-a841-8270c65f2a17.json b/evaluation_examples/examples/multi_apps/881deb30-9549-4583-a841-8270c65f2a17.json new file mode 100644 index 0000000..c6a138f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/881deb30-9549-4583-a841-8270c65f2a17.json @@ -0,0 +1,186 @@ +{ + "id": "881deb30-9549-4583-a841-8270c65f2a17", + "snapshot": "libreoffice_calc", + "instruction": "I want to find a faculty job in Hong Kong, so I am more curious about the \"Early Career Scheme\" of those schools is better to apply, please help me to count all the documents in the ecs pdf files in my hand, and organize the pass rate of each school by year into table!", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/Documents/Fundings/ecs", + "/home/user/Documents/Fundings/grf" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "/home/user/Documents/Fundings/ecs/ecs15.pdf", + "url": "https://drive.google.com/uc?id=1FTiT3mLlkehe2yWVdSMWr1w4ltLtQZUy&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs16.pdf", + "url": "https://drive.google.com/uc?id=1DMzZyhDey3lDsQ7fcPiJm9AYGECZms3q&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs17.pdf", + "url": "https://drive.google.com/uc?id=1TgAWk7FiV8fNrG2L3-Eu7BDccqtXebXY&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs23.pdf", + "url": "https://drive.google.com/uc?id=11DVxH4eRjECUxZNbUhYNJhgD0Y5WoN8r&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs22.pdf", + "url": "https://drive.google.com/uc?id=17IgyJADA65F40kH79S90QgEzPa7IERXx&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs21.pdf", + "url": "https://drive.google.com/uc?id=1kB4eFfLisPXKOirGUHbbcOyf73t7MVqL&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs20.pdf", + "url": "https://drive.google.com/uc?id=179j9tD1xRSgd9COM7rzErO6FLYO2sc_a&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs19.pdf", + "url": "https://drive.google.com/uc?id=1ogZUTfKjvQhg58GXtVUxe1U8VvHU-3ap&export=download" + }, + { + "path": "/home/user/Documents/Fundings/ecs/ecs18.pdf", + "url": "https://drive.google.com/uc?id=1MfuvRhAnhMEMbxn5js2ffWqMHJx2fsd2&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/customer-information-sheet-for-inward-payments-to-hong-kong.pdf", + "url": "https://drive.google.com/uc?id=1s-H3an7HLBM9ku6d6Hcdj1qkSwKAHngU&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf15.pdf", + "url": "https://drive.google.com/uc?id=1rRQXo9XHnCVTG8XqNAv0SJwPTW36MMbm&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf16.pdf", + "url": "https://drive.google.com/uc?id=18ljRqkdyXEZ464E0dpKjaEa2NFexyw3I&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf17.pdf", + "url": "https://drive.google.com/uc?id=1VrqOnyhpOkMpyIJ6YMrAhixpahjYonOd&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf18.pdf", + "url": "https://drive.google.com/uc?id=182CLDUr372-jpAiY4YvSbGNXF9TsWxzA&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf19.pdf", + "url": "https://drive.google.com/uc?id=1YkJtjlklKN0NmLiI2Hi4f_dKtTm5SPxT&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf20.pdf", + "url": "https://drive.google.com/uc?id=1a7Uc7VCMlEX6fy-5oqE6i1YLitBe7gaf&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf21.pdf", + "url": "https://drive.google.com/uc?id=1s8km4Wle4lc5PkbUQfivBFK0IJQgxMiB&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf22.pdf", + "url": "https://drive.google.com/uc?id=1HTEb1uK7LNvbVyeXgO8WemCPPgiASKiy&export=download" + }, + { + "path": "/home/user/Documents/Fundings/grf/grf23.pdf", + "url": "https://drive.google.com/uc?id=1XM-jZlfu_i4waDZHb8Z6Vr5b3LgULTtP&export=download" + }, + { + "path": "/home/user/Documents/Fundings/supported_rate.xlsx", + "url": "https://drive.google.com/uc?id=1jkC80XAlm4V_zZF4m75MGDqnVi9Pq-Ft&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Documents/Fundings/supported_rate.xlsx" + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Documents/Fundings/grf" + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Documents/Fundings/ecs" + } + } + ], + "trajectory": "trajectories/881deb30-9549-4583-a841-8270c65f2a17", + "related_apps": [ + "libreoffice_calc", + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", + "--outdir", + "/home/user/Documents/Fundings", + "/home/user/Documents/Fundings/supported_rate.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/Documents/Fundings/supported_rate.xlsx", + "/home/user/Documents/Fundings/supported_rate-Sheet1.csv" + ], + "dest": [ + "supported_rate.xlsx", + "supported_rate-Sheet1.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.google.com/uc?id=11mBEe1TnJhtHD5QolFuQhPvRCOSpm8ko&export=download", + "https://drive.google.com/uc?id=1len3Jz4cAztVC0t9dUyECCGecRrCsIHj&export=download" + ], + "dest": [ + "supported_rate-gt.xlsx", + "supported_rate-gt-Sheet1.csv" + ], + "multi": true + }, + "options": { + "rules": [ + { + "type": "sheet_print", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1" + } + ] + } + } +} diff --git a/evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json b/evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json index d45d4df..62b1c30 100644 --- a/evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json +++ b/evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json @@ -49,7 +49,8 @@ ], "trajectory": "trajectories/9219480b-3aed-47fc-8bac-d2cffc5849f7", "related_apps": [ - + "vs_code", + "os" ], "evaluator": { "postconfig": [], diff --git a/evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json b/evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json index 5161462..67d82b5 100644 --- a/evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json +++ b/evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json @@ -1,7 +1,7 @@ { "id": "d1acdb87-bb67-4f30-84aa-990e56a09c92", "snapshot": "libreoffice_calc", - "instruction": "Can you help me find these restaurants on Google Maps? I have a list of places I've been wanting to visit, and I need to gather some detailed information about them, including the address, what people have to say about them (ratings), and the hours of operation. It would be great if you could search for each of them on Google Maps and help me jot down this information into my sheet file, thanks!", + "instruction": "Hello! I'm eagerly planning a culinary adventure to Hong Kong and have curated a list of must-visit restaurants that I've been longing to explore. However, I could use some assistance in compiling a few essential details about these establishments. Would you be so kind as to help me out? It would be fantastic if you could search for these restaurants on Google Maps. I'm particularly interested in obtaining their addresses, any available websites, and contact phone numbers. If you could gather this information and input it into my form file, I would be immensely grateful. Many thanks in advance!", "source": "authors", "config": [ { @@ -52,13 +52,54 @@ "libreoffice_calc" ], "evaluator": { - "postconfig": [], - "func": "", + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", + "--outdir", + "/home/user/Desktop", + "/home/user/Desktop/MUST_VISIT.xlsx" + ] + } + } + ], + "func": "compare_table", "result": { + "type": "vm_file", + "path": [ + "/home/user/Desktop/MUST_VISIT.xlsx", + "/home/user/Desktop/MUST_VISIT-Sheet1.csv" + ], + "dest": [ + "MUST_VISIT.xlsx", + "MUST_VISIT-Sheet1.csv" + ], + "multi": true }, "expected": { + "type": "cloud_file", + "path": [ + "https://drive.google.com/uc?id=1MV6jBvRbbYwPqeFTd_nX40xzyltNhphl&export=download", + "https://drive.google.com/uc?id=1CGoRQDLw9-Ai7daq3qCz0o9kYSZB2WNn&export=download" + ], + "dest": [ + "MUST_VISIT-gt.xlsx", + "MUST_VISIT-gt-Sheet1.csv" + ], + "multi": true }, "options": { + "rules": [ + { + "type": "sheet_print", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1" + } + ] } } }