lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-29 21:53:12 +08:00
parent 343813a29b d8a497a417
commit 91876b2e00
2 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
View File

@@ -2,6 +2,7 @@ import xml.etree.ElementTree as ET
from PIL import Image, ImageDraw, ImageFont from PIL import Image, ImageDraw, ImageFont
from typing import Tuple
def find_leaf_nodes(xlm_file_str): def find_leaf_nodes(xlm_file_str):
if not xlm_file_str: if not xlm_file_str:
@@ -105,15 +106,20 @@ def draw_bounding_boxes(nodes, image_file_path, output_image_file_path):
# Draw index number at the bottom left of the bounding box with black background # Draw index number at the bottom left of the bounding box with black background
text_position = (coords[0], bottom_right[1]) # Adjust Y to be above the bottom right text_position = (coords[0], bottom_right[1]) # Adjust Y to be above the bottom right
draw.rectangle([text_position, (text_position[0] + 25, text_position[1] + 18)], fill='black') text_bbox: Tuple[int, int ,int ,int] = draw.textbbox(text_position, str(index), font=font, anchor="lb")
draw.text(text_position, str(index), font=font, fill="white") #offset: int = bottom_right[1]-text_bbox[3]
#text_bbox = (text_bbox[0], text_bbox[1]+offset, text_bbox[2], text_bbox[3]+offset)
#draw.rectangle([text_position, (text_position[0] + 25, text_position[1] + 18)], fill='black')
draw.rectangle(text_bbox, fill='black')
draw.text(text_position, str(index), font=font, anchor="lb", fill="white")
index += 1 index += 1
# each mark is an x, y, w, h tuple # each mark is an x, y, w, h tuple
marks.append([coords[0], coords[1], size[0], size[1]]) marks.append([coords[0], coords[1], size[0], size[1]])
drew_nodes.append(_node) drew_nodes.append(_node)
except ValueError as e: except ValueError:
pass pass
# Save the result # Save the result

1
requirements.txt
View File

@@ -38,3 +38,4 @@ pydrive
fastdtw fastdtw
openai openai
func-timeout