update multi-apps
This commit is contained in:
rhythmcao
@@ -29,7 +29,7 @@ from .chrome import (
|
||||
from .file import get_cloud_file, get_vm_file, get_cache_file, get_content_from_vm_file
|
||||
from .general import get_vm_command_line, get_vm_terminal_output
|
||||
from .gimp import get_gimp_config_file
|
||||
from .impress import get_audio_in_slide
|
||||
from .impress import get_audio_in_slide, get_background_image_in_slide
|
||||
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
|
||||
from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime
|
||||
from .replay import get_replay
|
||||
|
||||
@@ -7,6 +7,67 @@ from typing import Dict
|
||||
from desktop_env.evaluators.getters.file import get_vm_file
|
||||
|
||||
|
||||
def get_background_image_in_slide(env, config: Dict[str, str]):
|
||||
ppt_file_path, slide_index, dest = config["ppt_file_path"], int(config["slide_index"]), config["dest"]
|
||||
image_id, image_file_path = None, None
|
||||
|
||||
ppt_file_localhost_path = get_vm_file(env, {"path": ppt_file_path, "dest": os.path.split(ppt_file_path)[-1]})
|
||||
|
||||
with zipfile.ZipFile(ppt_file_localhost_path, 'r') as myzip:
|
||||
slide1_xml_file = 'ppt/slides/slide{}.xml'.format(slide_index + 1)
|
||||
# firstly, check whether the background image is used in the slide
|
||||
if slide1_xml_file not in myzip.namelist(): return None
|
||||
with myzip.open(slide1_xml_file) as f:
|
||||
# Parse the XML tree from the relationships file
|
||||
tree = ET.parse(f)
|
||||
root = tree.getroot()
|
||||
bg_tag = "{http://schemas.openxmlformats.org/presentationml/2006/main}bgPr"
|
||||
image_tag = "{http://schemas.openxmlformats.org/drawingml/2006/main}blip"
|
||||
attr_tag = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed"
|
||||
for child in root.iter(bg_tag):
|
||||
try:
|
||||
for element in child.iter(image_tag):
|
||||
image_id = element.attrib[attr_tag]
|
||||
break
|
||||
except: pass
|
||||
if image_id is not None: break
|
||||
else: return None
|
||||
|
||||
# next, extract the background image from the slide
|
||||
slide1_rels_file = 'ppt/slides/_rels/slide{}.xml.rels'.format(slide_index + 1)
|
||||
if slide1_rels_file in myzip.namelist():
|
||||
with myzip.open(slide1_rels_file) as f:
|
||||
# Parse the XML tree from the relationships file
|
||||
tree = ET.parse(f)
|
||||
root = tree.getroot()
|
||||
# Define the namespace used in the relationships file
|
||||
namespaces = {'r': 'http://schemas.openxmlformats.org/package/2006/relationships'}
|
||||
# Look for all relationship elements that have a type attribute for image
|
||||
for rel in root.findall('r:Relationship', namespaces):
|
||||
# Check if the relationship is for an image file
|
||||
if 'image' in rel.attrib['Type'] and rel.attrib['Id'] == image_id:
|
||||
target = rel.attrib['Target']
|
||||
if target.startswith('..'):
|
||||
# Resolve the relative path to get the correct path within the zip file
|
||||
image_file_path = os.path.normpath(os.path.join('ppt/slides', target))
|
||||
# Replace backslashes with forward slashes for ZIP compatibility
|
||||
image_file_path = image_file_path.replace('\\', '/')
|
||||
tmpdirname = os.path.dirname(ppt_file_localhost_path)
|
||||
myzip.extract(image_file_path, tmpdirname)
|
||||
image_file_path = os.path.join(tmpdirname, image_file_path)
|
||||
return image_file_path
|
||||
else: # absolute path
|
||||
assert target.startswith("file://"), target
|
||||
image_file_path = target[7:]
|
||||
break
|
||||
if image_file_path is None:
|
||||
return None
|
||||
|
||||
else:
|
||||
# Get the audio file from vm and return the file path in the host
|
||||
return get_vm_file(env, {"path": image_file_path, "dest": dest})
|
||||
|
||||
|
||||
def get_audio_in_slide(env, config: Dict[str, str]):
|
||||
ppt_file_path, slide_index, dest = config["ppt_file_path"], int(config["slide_index"]), config["dest"]
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import re
|
||||
from typing import Dict
|
||||
|
||||
|
||||
@@ -86,6 +87,18 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
|
||||
with open(expected) as f2:
|
||||
expected_text = f2.read()
|
||||
|
||||
ignore_blanks = options.get('ignore_blanks', False)
|
||||
if ignore_blanks:
|
||||
actual_text = re.sub(r'[\t\n]', ' ', actual_text).strip()
|
||||
actual_text = re.sub(r'\s+', ' ', actual_text)
|
||||
expected_text = re.sub(r'[\t\n]', ' ', expected_text).strip()
|
||||
expected_text = re.sub(r'\s+', ' ', expected_text)
|
||||
|
||||
ignore_case = options.get('ignore_case', False)
|
||||
if ignore_case:
|
||||
actual_text = actual_text.lower()
|
||||
expected_text = expected_text.lower()
|
||||
|
||||
if actual_text == expected_text:
|
||||
return 1.0
|
||||
return 0.0
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"id": "0e5303d4-8820-42f6-b18d-daf7e633de21",
|
||||
"snapshot": "chrome",
|
||||
"instruction": "I want to learn python programming and my friend recommends me this course website. I have grabbed the lecture slide for week 0. Please download the PDFs for other weeks into the opened folder and leave the file name as-it-is.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://cs50.harvard.edu/python/2022/weeks/0/"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"mkdir",
|
||||
"-p",
|
||||
"/home/user/lecture_slides"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1OdvHgcHXSn62xXe_VrPTN0HLWHmrcfdY&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/lecture_slides/lecture0.pdf"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"nautilus",
|
||||
"/home/user/lecture_slides"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"os",
|
||||
"chrome"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig": [
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
"cd /home/user && zip -qr lecture_slides.zip lecture_slides/"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "compare_archive",
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/lecture_slides.zip",
|
||||
"dest": "lecture_slides.zip"
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.usercontent.google.com/download?id=1Ej2iHG8p-QJe7FZQpPIIS82BHOlFAUQM&export=download&authuser=0&confirm=t",
|
||||
"dest": "gold_lecture_slides.zip"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "236833a3-5704-47fc-888c-4f298f09f799",
|
||||
"snapshot": "chrome",
|
||||
"instruction": "Find daily papers on Huggingface and take down all the titles, authors and the abstracts of papers on 1st March, 2024 in the doc file 'paper_reading_2024_03_01.docx' on desktop. Each paragraph (split by empty lines) conforms to the following format:\nTitle: xxx\nAuthors: xxx, xxx, xxx\nAbstract: xxxxxxxx.\nArxiv PDF: https://xxxx.pdf",
|
||||
"instruction": "Find the daily paper list on Huggingface and take down the meta information of papers on 1st March, 2024 in the opened .docx file. I have recorded two papers. Please conform to the format and complete others.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
@@ -31,12 +31,24 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1WEDfILO-NijZBGArZ3ovO1933uHeOi1A&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/Desktop/paper_reading_2024_03_01.docx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"libreoffice",
|
||||
"--writer"
|
||||
"--writer",
|
||||
"/home/user/Desktop/paper_reading_2024_03_01.docx"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -55,7 +67,7 @@
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.usercontent.google.com/download?id=1TUTihXD93bIlekuYy_44fmXAhI1KVol4&export=download&authuser=0&confirm=t",
|
||||
"path": "https://drive.usercontent.google.com/download?id=1wb0sQnVDCAz8sS49kO8boJIa1kqI5mx0&export=download&authuser=0&confirm=t",
|
||||
"dest": "gold_paper_reading_2024_03_01.docx"
|
||||
},
|
||||
"options": {
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"id": "3e3fc409-bff3-4905-bf16-c968eee3f807",
|
||||
"snapshot": "chrome",
|
||||
"instruction": "I'm a huge movie fan and have kept a record of all the movies I've watched. I'm curious to find out if there are any films released before 2024 from the IMDB Top 30 list that I haven't seen yet. Help me create another sheet 'unseen_movies' in the opened Excel. This sheet should share the same headers and sort the results according to IMDB rankings from high to low.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://www.imdb.com"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1KVNVf5qZhprV_7rgEl33Qrkagv603reM&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/Desktop/movies.xlsx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"libreoffice",
|
||||
"--calc",
|
||||
"/home/user/Desktop/movies.xlsx"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"libreoffice_calc",
|
||||
"chrome"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "compare_table",
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/movies.xlsx",
|
||||
"dest": "movies.xlsx"
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.usercontent.google.com/download?id=149QKswQ8AIYk21Aaatic6QSCcBU40uyd&export=download&authuser=0&confirm=t",
|
||||
"dest": "gold_movies.xlsx"
|
||||
},
|
||||
"options": {
|
||||
"rules": [
|
||||
{
|
||||
"type": "sheet_data",
|
||||
"sheet_idx0": "RNunseen_movies",
|
||||
"sheet_idx1": "ENunseen_movies"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"id": "47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5",
|
||||
"snapshot": "vlc",
|
||||
"instruction": "The landscape at 00:08 in this video is so beautiful. Please extract this frame and set it as the background of the second page of the opened slides.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1-De7kW-JOEiwlZXfLG7bSTpycUz7zw_Y&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/Desktop/landscape.mp4"
|
||||
},
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1v7J3uaxjM5wSz8xidcV0p7tMfkJCHw3_&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/Desktop/Robotic_Workshop_Infographics.pptx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "open",
|
||||
"parameters": {
|
||||
"path": "/home/user/Desktop/Robotic_Workshop_Infographics.pptx"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"vlc",
|
||||
"--repeat",
|
||||
"/home/user/Desktop/landscape.mp4"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"libreoffice_impress",
|
||||
"vlc"
|
||||
],
|
||||
"evaluator": {
|
||||
"postconfig": [
|
||||
{
|
||||
"type": "activate_window",
|
||||
"parameters": {
|
||||
"window_name": "Robotic_Workshop_Infographics.pptx - LibreOffice Impress",
|
||||
"strict": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sleep",
|
||||
"parameters": {
|
||||
"seconds": 0.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"python",
|
||||
"-c",
|
||||
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sleep",
|
||||
"parameters": {
|
||||
"seconds": 0.5
|
||||
}
|
||||
}
|
||||
],
|
||||
"func": "compare_images",
|
||||
"result": {
|
||||
"type": "background_image_in_slide",
|
||||
"ppt_file_path": "/home/user/Desktop/Robotic_Workshop_Infographics.pptx",
|
||||
"slide_index": 1,
|
||||
"dest": "landscape.png"
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.usercontent.google.com/download?id=17Dt8EalJj9ksbfuj6dowY3OpDqcK80QZ&export=download&authuser=0&confirm=t",
|
||||
"dest": "gold_landscape.png"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"id": "df67aebb-fb3a-44fd-b75b-51b6012df509",
|
||||
"snapshot": "vscode",
|
||||
"instruction": "I am writing my paper thesis. I have listed all referenced papers in the opened docx. But my mentor asked me to use latex instead of word writer. So could you help me export the dblp bibtex of these papers into 'references.bib'. By the way, if the paper is published, do not use the arxiv version. Separate each bibtex dict with a blank line for clarity.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://dblp.org/"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"touch",
|
||||
"/home/user/Desktop/references.bib"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"code",
|
||||
"/home/user/Desktop/references.bib"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1NDYJ7XyiUVJYuMBDAYvObrSZdODrhUMi&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/Desktop/references.docx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"libreoffice",
|
||||
"--writer",
|
||||
"/home/user/Desktop/references.docx"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome",
|
||||
"libreoffice_writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "compare_text_file",
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/references.bib",
|
||||
"dest": "references.bib"
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.usercontent.google.com/download?id=1eeaPRSAWhzsNGFuda_u6phsdQMZHBkb2&export=download&authuser=0&confirm=t",
|
||||
"dest": "gold_references.bib"
|
||||
},
|
||||
"options": {
|
||||
"ignore_blanks": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"id": "df67aebb-fb3a-44fd-b75b-51b6012df509",
|
||||
"snapshot": "vscode",
|
||||
"instruction": "I am writing my paper thesis. I have listed all referenced papers in the opened docx. But my mentor asked me to use latex instead of word writer. So could you help me export the dblp bibtex of these papers into 'references.bib'. By the way, if the paper is published, do not use the arxiv version. Separate each bibtex dict with a blank line for clarity.",
|
||||
"source": "authors",
|
||||
"config": [
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"google-chrome",
|
||||
"--remote-debugging-port=1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"socat",
|
||||
"tcp-listen:9222,fork",
|
||||
"tcp:localhost:1337"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "chrome_open_tabs",
|
||||
"parameters": {
|
||||
"urls_to_open": [
|
||||
"https://dblp.org/"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "execute",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"touch",
|
||||
"/home/user/Desktop/references.bib"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"code",
|
||||
"/home/user/Desktop/references.bib"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "download",
|
||||
"parameters": {
|
||||
"files": [
|
||||
{
|
||||
"url": "https://drive.usercontent.google.com/download?id=1NDYJ7XyiUVJYuMBDAYvObrSZdODrhUMi&export=download&authuser=0&confirm=t",
|
||||
"path": "/home/user/Desktop/references.docx"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "launch",
|
||||
"parameters": {
|
||||
"command": [
|
||||
"libreoffice",
|
||||
"--writer",
|
||||
"/home/user/Desktop/references.docx"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"trajectory": "trajectories/",
|
||||
"related_apps": [
|
||||
"chrome",
|
||||
"libreoffice_writer"
|
||||
],
|
||||
"evaluator": {
|
||||
"func": "compare_text_file",
|
||||
"result": {
|
||||
"type": "vm_file",
|
||||
"path": "/home/user/Desktop/references.bib",
|
||||
"dest": "references.bib"
|
||||
},
|
||||
"expected": {
|
||||
"type": "cloud_file",
|
||||
"path": "https://drive.usercontent.google.com/download?id=1eeaPRSAWhzsNGFuda_u6phsdQMZHBkb2&export=download&authuser=0&confirm=t",
|
||||
"dest": "gold_references.bib"
|
||||
},
|
||||
"options": {
|
||||
"ignore_blanks": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user