lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan26th

Browse Source 
fixed the too long time consuming of massy AT nodes
added special handling for libreoffice calc
This commit is contained in:
David Chang
2024-01-26 12:30:41 +08:00
parent 147b7647b3
commit 78c7b702b3
1 changed files with 59 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
desktop_env/server/main.py
View File

@@ -181,7 +181,7 @@ _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
}
def _create_atspi_node(node: Accessible) -> _Element:
def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) -> _Element:
attribute_dict: Dict[str, Any] = {"name": node.name}
# States {{{ #
@@ -302,15 +302,58 @@ def _create_atspi_node(node: Accessible) -> _Element:
)
if "text" in locals() and len(text) > 0:
xml_node.text = text
# HACK: libreoffice has a problem -> billions of children for parent with RoleName "document spreadsheet"
if node.getRoleName() == "document spreadsheet":
#if node.getRoleName() == "document spreadsheet":
#return xml_node
# HYPERPARAMETER
if depth==50:
logger.warning("Max depth reached")
return xml_node
for ch in node:
xml_node.append(_create_atspi_node(ch))
return xml_node
# HACK: libreoffice has a problem -> billions of children for parent with RoleName "document spreadsheet"
if node.getRoleName() == "document spreadsheet":
flag = "calc"
if flag=="calc" and node_role_name=="table":
# Maximum column: 1024 if ver<=7.3 else 16384
# Maximum row: 104 8576
# Maximun sheet: 1 0000
version_str: str = subprocess.run("libreoffice --version", shell=True, text=True, stdout=subprocess.PIPE).stdout
version_str = version_str.split()[1]
version_tuple: Tuple[int] = tuple(map(int, version_str.split(".")))
MAXIMUN_COLUMN = 1024 if version_tuple<(7, 4) else 16384
MAX_ROW = 104_8576
index_base = 0
first_showing = False
column_base = None
for r in range(MAX_ROW):
#logger.warning(r)
for clm in range(column_base or 0, MAXIMUN_COLUMN):
child_node: Accessible = node[index_base+clm]
showing: bool = child_node.getState().contains(STATE_SHOWING)
if showing:
child_node: _Element = _create_atspi_node(child_node, depth+1, flag)
if not first_showing:
column_base = clm
first_showing = True
xml_node.append(child_node)
elif first_showing and column_base is not None or clm>=500:
break
if first_showing and clm==column_base or not first_showing and r>=500:
break
index_base += MAXIMUN_COLUMN
return xml_node
else:
for i, ch in enumerate(node):
# HYPERPARAMETER
if i>=1025:
logger.warning("Max width reached")
break
xml_node.append(_create_atspi_node(ch, depth+1, flag))
return xml_node
@app.route("/accessibility", methods=["GET"])
def get_accessibility_tree():
@@ -319,8 +362,9 @@ def get_accessibility_tree():
# AT-SPI works for KDE as well
if os_name == "Linux":
desktop: Accessible = pyatspi.Registry.getDesktop(0)
desktop_xml: _Element = _create_atspi_node(desktop)
desktop_xml: _Element = _create_atspi_node(desktop, 0)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
# TODO: Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned
else:
return "Currently not implemented for platform {:}.".format(platform.platform()), 500
@@ -487,7 +531,7 @@ def get_directory_tree():
def get_file():
# Retrieve filename from the POST request
if 'file_path' in request.form:
file_path = os.path.expanduser(request.form['file_path'])
file_path = os.path.expandvars(os.path.expanduser(request.form['file_path']))
else:
return jsonify({"error": "file_path is required"}), 400
@@ -503,7 +547,7 @@ def get_file():
def upload_file():
# Retrieve filename from the POST request
if 'file_path' in request.form and 'file_data' in request.files:
file_path = os.path.expanduser(request.form['file_path'])
file_path = os.path.expandvars(os.path.expanduser(request.form['file_path']))
file = request.files["file_data"]
file.save(file_path)
return "File Uploaded"
@@ -529,7 +573,7 @@ def change_wallpaper():
if not path:
return "Path not supplied!", 400
path = Path(os.path.expanduser(path))
path = Path(os.path.expandvars(os.path.expanduser(path)))
if not path.exists():
return f"File not found: {path}", 404
@@ -560,10 +604,11 @@ def download_file():
if not url or not path:
return "Path or URL not supplied!", 400
path = Path(os.path.expanduser(path))
path = Path(os.path.expandvars(os.path.expanduser(path)))
path.parent.mkdir(parents=True, exist_ok=True)
max_retries = 3
error: Optional[Exception] = None
for i in range(max_retries):
try:
response = requests.get(url, stream=True)
@@ -576,9 +621,10 @@ def download_file():
return "File downloaded successfully"
except requests.RequestException as e:
error = e
logger.error(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)")
return f"Failed to download {url}. No retries left. Error: {e}", 500
return f"Failed to download {url}. No retries left. Error: {error}", 500
@app.route("/setup/open_file", methods=['POST'])
@@ -589,7 +635,7 @@ def open_file():
if not path:
return "Path not supplied!", 400
path = Path(os.path.expanduser(path))
path = Path(os.path.expandvars(os.path.expanduser(path)))
if not path.exists():
return f"File not found: {path}", 404