From d6cd0936b3c317edd6317736725658cbc7a15620 Mon Sep 17 00:00:00 2001 From: David Chang Date: Thu, 7 Mar 2024 16:54:06 +0800 Subject: [PATCH] ver Mar7th updated instructions and set-up configs --- desktop_env/evaluators/metrics/general.py | 5 ++- desktop_env/evaluators/metrics/others.py | 2 +- .../00fa164e-2612-4439-992e-157d019a8436.json | 2 +- .../3f05f3b9-29ba-4b6b-95aa-2204697ffc06.json | 38 +++++++++++++++- .../415ef462-bed3-493a-ac36-ca8c6d23bf1b.json | 8 +++- .../42d25c08-fb87-4927-8b65-93631280a26f.json | 2 +- .../48c46dc7-fe04-4505-ade7-723cba1aa6f6.json | 2 +- .../74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json | 44 ++++++++++++++++++- .../788b3701-3ec9-4b67-b679-418bfa726c22.json | 10 ++++- .../b5062e3e-641c-4e3a-907b-ac864d2e7652.json | 8 +++- .../f5c13cdd-205c-4719-a562-348ae5cd1d91.json | 20 ++++++--- 11 files changed, 121 insertions(+), 20 deletions(-) diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 26a8e3c..3e15edb 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -17,6 +17,9 @@ import difflib from .utils import _match_record, _match_value_to_rule +import logging + +logger = logging.getLogger("desktopenv.metric.general") def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float: if result is None: @@ -166,7 +169,7 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float: raise ValueError("At least one of xpath and selectors is required") if len(elements) == 0: - print("no elements") + logger.info("No elements: %s", r["xpath"] if "xpath" in r else r["selectors"]) return 0. if "text" in r: diff --git a/desktop_env/evaluators/metrics/others.py b/desktop_env/evaluators/metrics/others.py index 7892d0e..706bcf3 100644 --- a/desktop_env/evaluators/metrics/others.py +++ b/desktop_env/evaluators/metrics/others.py @@ -81,7 +81,7 @@ def check_mp3_meta(result: str, meta: Dict[str, Dict[str, Union[str, V]]]) -> bo for k, r in meta.items(): value = id3_dict.get(k, "") if isinstance(value, list): - value: str = value[0] + value: str = ",".join(value) logger.debug("%s.%s: %s", result, k, value) metric = metric and _match_value_to_rule(value, r) return float(metric) diff --git a/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json b/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json index 3baf77f..a329b4c 100644 --- a/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json +++ b/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json @@ -1,7 +1,7 @@ { "id": "00fa164e-2612-4439-992e-157d019a8436", "snapshot": "libreoffice_writer", - "instruction": "I'm writing a report in Writer. A little data are recorded in \"~/Documents/awesome-desktop/expe-results.xlsx\". Help me to extract the results of LLM-based models and insert a table in the \"Main Results\" section of my report.", + "instruction": "I need to include the experiment results from \"~/Documents/awesome-desktop/expe-results.xlsx\" into the currently writing report. Specifically, extract the results of LLM-based models and insert a table into the \"Main Results\" section of my report.", "source": "authors", "config": [ { diff --git a/evaluation_examples/examples/multi_apps/3f05f3b9-29ba-4b6b-95aa-2204697ffc06.json b/evaluation_examples/examples/multi_apps/3f05f3b9-29ba-4b6b-95aa-2204697ffc06.json index 60d0a16..0a82fbc 100644 --- a/evaluation_examples/examples/multi_apps/3f05f3b9-29ba-4b6b-95aa-2204697ffc06.json +++ b/evaluation_examples/examples/multi_apps/3f05f3b9-29ba-4b6b-95aa-2204697ffc06.json @@ -1,7 +1,7 @@ { "id": "3f05f3b9-29ba-4b6b-95aa-2204697ffc06", "snapshot": "os", - "instruction": "I have a collection of MP3s named with their artist and title. However, their meta data are messy. I don't know how to quickly fix them. Some people tells that Picard or Kid3 may help, but I have never used them. Please help me to do this.", + "instruction": "I have a collection of MP3s with blank meta data, but already named with their artists and titles. I've heard that Picard or Kid3 may help, but I'm unfamiliar with them. Can you help me to fix the meta data?", "source": "authors", "config": [ { @@ -49,6 +49,40 @@ "parameters": { "command": ["tar", "-xJvf", "/home/user/.pass-through.tar.xz", "-C", "/home/user/Documents/Novels/Pass Through"] } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://picard.musicbrainz.org/", + "https://kid3.kde.org/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": ["nautilus", "/home/user/Music"] + } } ], "trajectory": "trajectories/3f05f3b9-29ba-4b6b-95aa-2204697ffc06", @@ -79,7 +113,7 @@ { "type": "vm_file", "path": "/home/user/Music/Zhou Xuan - Nights in Shanghai.mp3", - "dest": "/Zhou Xuan - Nights in Shanghai.mp3" + "dest": "Zhou Xuan - Nights in Shanghai.mp3" } ], "expected": [ diff --git a/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json b/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json index f3365fb..be80232 100644 --- a/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json +++ b/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json @@ -1,7 +1,7 @@ { "id": "415ef462-bed3-493a-ac36-ca8c6d23bf1b", "snapshot": "thunderbird", - "instruction": "Save the AWS invoice of December from the email. I have moved that email to local \"Bills\" folder. Save it to the my receipts folder. Keep the file name pattern and update a record to my tally book.", + "instruction": "There's an e-mail containing the AWS invoice for December saved in local \"Bills\" folder. Extract the invoice PDF to the my receipts folder. Follow the file name pattern of the old files and update a record in my tally book.", "source": "authors", "config": [ { @@ -53,6 +53,12 @@ "parameters": { "command": ["thunderbird"] } + }, + { + "type": "launch", + "parameters": { + "command": ["nautilus"] + } } ], "trajectory": "trajectories/415ef462-bed3-493a-ac36-ca8c6d23bf1b", diff --git a/evaluation_examples/examples/multi_apps/42d25c08-fb87-4927-8b65-93631280a26f.json b/evaluation_examples/examples/multi_apps/42d25c08-fb87-4927-8b65-93631280a26f.json index 7613590..f22de37 100644 --- a/evaluation_examples/examples/multi_apps/42d25c08-fb87-4927-8b65-93631280a26f.json +++ b/evaluation_examples/examples/multi_apps/42d25c08-fb87-4927-8b65-93631280a26f.json @@ -1,7 +1,7 @@ { "id": "42d25c08-fb87-4927-8b65-93631280a26f", "snapshot": "chrome", - "instruction": "Hey, my friend has just sent me a web novel, but in txt files. Could you please help me convert it to epub format, so that I can easily read it on my mobile and kindle? I have found several tools to obtain an epub file, like https://github.com/kevinboone/txt2epub and https://github.com/potatoeggy/noveldown. But I'm not sure which one helps. Please help me to do this. Remember to name the file with novel's title.", + "instruction": "Hey, my friend has just sent me a web novel, but in txt files. Could you please help me to convert it to epub format for easy reading on my mobile and kindle? I have found several tools to obtain an epub file, opened in the browser. But I'm not sure which one helps. Please help me to do this and ensure to name the file with novel's title.", "source": "authors", "config": [ { diff --git a/evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json b/evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json index 8c5ef7f..53f335a 100644 --- a/evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json +++ b/evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json @@ -1,7 +1,7 @@ { "id": "48c46dc7-fe04-4505-ade7-723cba1aa6f6", "snapshot": "chrome", - "instruction": "I'm currently working on OSWorld project. Every time I need to open the terminal and change to the project directory, open the file manager and change to the project directory, and open relative webpages in chrome browser. This is tedious. Help me to do this automatically. Open project directory in terminal and file manager and open github and online document in chrome browser. That's enough.", + "instruction": "Help me to automatically set up my work space. To be specific, open project directory of OSWorld in terminal and file manager, then open github homepage and the online document of Python in chrome browser.", "source": "authors", "config": [ { diff --git a/evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json b/evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json index 731020a..e3908bf 100644 --- a/evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json +++ b/evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json @@ -1,9 +1,49 @@ { "id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41", "snapshot": "chrome", - "instruction": "Help me to get an initial setup of web extension with help of https://webext.eu . Tag the extension as \"happy-extension v0.0.1\". Leave description as blank for now. A background script and browser action is needed, while other features are not. Place the auto-generated folder under folder \"~/Projects\".", + "instruction": "Help me to set up an initial web extension project with help of the web tool, tagging it \"happy-extension v0.0.1\". Leave description blank for now. Include a background script and browser action, while other features are not required. Remember to unzip the auto-generated folder into \"~/Projects\".", "source": "authors", - "config": [], + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://webext.eu" + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["mkdir", "-p", "/home/user/Projects"] + } + }, + { + "type": "launch", + "parameters": { + "command": ["nautilus", "/home/user/Projects"] + } + } + ], "trajectory": "trajectories/74d5859f-ed66-4d3e-aa0e-93d7a592ce41", "related_apps": [ "chrome", diff --git a/evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json b/evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json index 4b1da9f..cd83927 100644 --- a/evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json +++ b/evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json @@ -1,7 +1,7 @@ { "id": "788b3701-3ec9-4b67-b679-418bfa726c22", "snapshot": "chrome", - "instruction": "I'm following a small tale set updating on https://github.com/4th-year-in-tsinghua/4th-year-in-tsinghua. I have already downloaded several chapters to my computer for reading and archiving. There should be some updates and help me to download the next chapter I haven't downloaded yet. Save it to my novel collection folder.", + "instruction": "I'm tracking updates for a short tale set on https://github.com/liangjs333/4th-year-in-tsinghua-eng. I have already downloaded several chapters for reading and archiving. Please assist me in downloading the next chapter I haven't obtained yet and save it to my novel collection folder.", "source": "authors", "config": [ { @@ -37,6 +37,12 @@ "parameters": { "command": ["tar", "-xJvf", "/home/user/.old-chapters.tar.xz", "-C", "/home/user/Documents/Novels/4th Year in Tsinghua"] } + }, + { + "type": "launch", + "parameters": { + "command": ["nautilus", "/home/user/Documents/Novels/4th Year in Tsinghua"] + } } ], "trajectory": "trajectories/788b3701-3ec9-4b67-b679-418bfa726c22", @@ -45,7 +51,7 @@ "func": "diff_text_file", "result": { "type": "vm_file", - "path": "/home/user/Documents/Novels/4th Year in Tsinghua/早期建筑群.tex", + "path": "/home/user/Documents/Novels/4th Year in Tsinghua/Early Buildings.tex", "dest": "download.tex" }, "expected": { diff --git a/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json b/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json index 99c4da3..5d84224 100644 --- a/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json +++ b/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json @@ -1,7 +1,7 @@ { "id": "b5062e3e-641c-4e3a-907b-ac864d2e7652", "snapshot": "libreoffice_calc", - "instruction": "I've got a mass of papers in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".", + "instruction": "Please help me to extract the name, e-mail, and affiliation of the first author from each paper in the folder and organize them in an Excel table. Include headers for each field. Sort the authors by their full names alphabetically and save the table as \"~/authors.xlsx\".", "source": "authors", "config": [ { @@ -32,6 +32,12 @@ } ] } + }, + { + "type": "execute", + "parameters": { + "command": ["nautilus", "/home/user/Documents/Papers"] + } } ], "trajectory": "trajectories/b5062e3e-641c-4e3a-907b-ac864d2e7652", diff --git a/evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json b/evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json index 131f65a..582a9a6 100644 --- a/evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json +++ b/evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json @@ -1,7 +1,7 @@ { "id": "f5c13cdd-205c-4719-a562-348ae5cd1d91", "snapshot": "thunderbird", - "instruction": "Here is a table recording tuition payments in ~/Documents/Departments/finance. I have already composed an e-mail to remind people who haven't finished payment yet. Please help me to add their emails to the receiver field.", + "instruction": "I've drafted an e-mail reminder for those who haven't paid tuition. Please help me to check out their e-mails from the payment record and add to the receiver field.", "source": "authors", "config": [ { @@ -24,7 +24,7 @@ }, { "url": "https://drive.google.com/uc?id=1nNc0NoOuP3Of0eGsKY-1kctg63vIjXl5&export=download", - "path": "/home/user/Documents/Departments/finance/tuition.xlsx" + "path": "/home/user/Documents/Departments/finance/tuition_payment.xlsx" } ] } @@ -49,7 +49,13 @@ "command": "/usr/bin/thunderbird -compose \"from='Anonym Tester ',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"", "shell": true } - } + }, + { + "type": "launch", + "parameters": { + "command": ["nautilus", "/home/user/Documents/Departments/finance"] + } + } ], "trajectory": "trajectories/f5c13cdd-205c-4719-a562-348ae5cd1d91", "related_apps": ["thunderbird", "os", "libreoffice_calc"], @@ -61,10 +67,10 @@ "expected": { "type": "rule", "rules": [ - {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"]}, - {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"]}, - {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"]}, - {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"]} + {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"]}, + {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"]}, + {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"]}, + {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"]} ] } }