lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-26 12:33:49 +08:00
parent f0b073989a 78c7b702b3
commit 4fbdcb56d2
2 changed files with 61 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
desktop_env/server/main.py
View File

@@ -181,7 +181,7 @@ _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
}
def _create_atspi_node(node: Accessible) -> _Element:
def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) -> _Element:
attribute_dict: Dict[str, Any] = {"name": node.name}
# States {{{ #
@@ -302,15 +302,58 @@ def _create_atspi_node(node: Accessible) -> _Element:
)
if "text" in locals() and len(text) > 0:
xml_node.text = text
# HACK: libreoffice has a problem -> billions of children for parent with RoleName "document spreadsheet"
if node.getRoleName() == "document spreadsheet":
#if node.getRoleName() == "document spreadsheet":
#return xml_node
# HYPERPARAMETER
if depth==50:
logger.warning("Max depth reached")
return xml_node
for ch in node:
xml_node.append(_create_atspi_node(ch))
return xml_node
# HACK: libreoffice has a problem -> billions of children for parent with RoleName "document spreadsheet"
if node.getRoleName() == "document spreadsheet":
flag = "calc"
if flag=="calc" and node_role_name=="table":
# Maximum column: 1024 if ver<=7.3 else 16384
# Maximum row: 104 8576
# Maximun sheet: 1 0000
version_str: str = subprocess.run("libreoffice --version", shell=True, text=True, stdout=subprocess.PIPE).stdout
version_str = version_str.split()[1]
version_tuple: Tuple[int] = tuple(map(int, version_str.split(".")))
MAXIMUN_COLUMN = 1024 if version_tuple<(7, 4) else 16384
MAX_ROW = 104_8576
index_base = 0
first_showing = False
column_base = None
for r in range(MAX_ROW):
#logger.warning(r)
for clm in range(column_base or 0, MAXIMUN_COLUMN):
child_node: Accessible = node[index_base+clm]
showing: bool = child_node.getState().contains(STATE_SHOWING)
if showing:
child_node: _Element = _create_atspi_node(child_node, depth+1, flag)
if not first_showing:
column_base = clm
first_showing = True
xml_node.append(child_node)
elif first_showing and column_base is not None or clm>=500:
break
if first_showing and clm==column_base or not first_showing and r>=500:
break
index_base += MAXIMUN_COLUMN
return xml_node
else:
for i, ch in enumerate(node):
# HYPERPARAMETER
if i>=1025:
logger.warning("Max width reached")
break
xml_node.append(_create_atspi_node(ch, depth+1, flag))
return xml_node
@app.route("/accessibility", methods=["GET"])
def get_accessibility_tree():
@@ -319,8 +362,9 @@ def get_accessibility_tree():
# AT-SPI works for KDE as well
if os_name == "Linux":
desktop: Accessible = pyatspi.Registry.getDesktop(0)
desktop_xml: _Element = _create_atspi_node(desktop)
desktop_xml: _Element = _create_atspi_node(desktop, 0)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
# TODO: Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned
else:
return "Currently not implemented for platform {:}.".format(platform.platform()), 500
@@ -487,7 +531,7 @@ def get_directory_tree():
def get_file():
# Retrieve filename from the POST request
if 'file_path' in request.form:
file_path = os.path.expanduser(request.form['file_path'])
file_path = os.path.expandvars(os.path.expanduser(request.form['file_path']))
else:
return jsonify({"error": "file_path is required"}), 400
@@ -503,7 +547,7 @@ def get_file():
def upload_file():
# Retrieve filename from the POST request
if 'file_path' in request.form and 'file_data' in request.files:
file_path = os.path.expanduser(request.form['file_path'])
file_path = os.path.expandvars(os.path.expanduser(request.form['file_path']))
file = request.files["file_data"]
file.save(file_path)
return "File Uploaded"
@@ -529,7 +573,7 @@ def change_wallpaper():
if not path:
return "Path not supplied!", 400
path = Path(os.path.expanduser(path))
path = Path(os.path.expandvars(os.path.expanduser(path)))
if not path.exists():
return f"File not found: {path}", 404
@@ -560,10 +604,11 @@ def download_file():
if not url or not path:
return "Path or URL not supplied!", 400
path = Path(os.path.expanduser(path))
path = Path(os.path.expandvars(os.path.expanduser(path)))
path.parent.mkdir(parents=True, exist_ok=True)
max_retries = 3
error: Optional[Exception] = None
for i in range(max_retries):
try:
response = requests.get(url, stream=True)
@@ -576,9 +621,10 @@ def download_file():
return "File downloaded successfully"
except requests.RequestException as e:
error = e
logger.error(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)")
return f"Failed to download {url}. No retries left. Error: {e}", 500
return f"Failed to download {url}. No retries left. Error: {error}", 500
@app.route("/setup/open_file", methods=['POST'])
@@ -589,7 +635,7 @@ def open_file():
if not path:
return "Path not supplied!", 400
path = Path(os.path.expanduser(path))
path = Path(os.path.expandvars(os.path.expanduser(path)))
if not path.exists():
return f"File not found: {path}", 404

4
main.py
View File

@@ -91,12 +91,12 @@ def human_agent():
logger.info("The episode is done.")
break
input("PAUSING")
#input("PAUSING")
result = env.evaluate()
logger.info("Result: %.2f", result)
input("PAUSING")
#input("PAUSING")
# env.close()
logger.info("Environment closed.")