lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix the cursor logo placement for ubuntu

Browse Source
This commit is contained in:
Timothyxxx
2023-12-28 15:37:18 +08:00
parent 2a9e5cc373
commit 9f0898fbd5
5 changed files with 202 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
desktop_env/evaluators/metrics/table.py
View File

@@ -1,14 +1,13 @@
import pandas as pd
#import pylightxl
import openpyxl
#from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet
from utils import load_charts, load_sparklines
from .utils import load_charts, load_sparklines
from typing import Dict, List
from typing import Any
def compare_table(actual, expected):
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
@@ -16,6 +15,7 @@ def compare_table(actual, expected):
# Compare the DataFrames
return 1 if df1.equals(df2) else 0
def compare_with_sparklines(actual: str, expected: str) -> float:
df1 = pd.read_excel(actual)
df2 = pd.read_excel(expected)
@@ -29,6 +29,7 @@ def compare_with_sparklines(actual: str, expected: str) -> float:
return float(normal_content_metric and sparkline_metric)
def compare_with_charts(actual: str, expected: str, **options) -> float:
"""
Args:
@@ -45,25 +46,26 @@ def compare_with_charts(actual: str, expected: str, **options) -> float:
charts1 = load_charts(actual, **options)
charts2 = load_charts(expected, **options)
chart_metric: bool = charts1==charts2
chart_metric: bool = charts1 == charts2
print("Chart Metric: {:}".format(chart_metric))
return float(normal_content_metric and chart_metric)
def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
#workbook: Workbook = openpyxl.load_workbook(filename=result)
# workbook: Workbook = openpyxl.load_workbook(filename=result)
workbook = pd.ExcelFile(result)
worksheet_names: List[str] = workbook.sheet_names
passes = True
for r in rules:
if r["type"]=="sheet_name":
if r["type"] == "sheet_name":
expected_name: str = worksheet_names[r["sheet_idx"]]
actual_name: str = r["sheet_name"]
metric: bool = expected_name==actual_name
metric: bool = expected_name == actual_name
print("Assertion: {:d}.{:} is {:} - {:}".format(r["sheet_idx"], actual_name, expected_name, metric))
passes = passes and metric
elif r["type"]=="sheet_data":
elif r["type"] == "sheet_data":
if isinstance(r["sheet_idx0"], int):
df1: pd.DataFrame = pd.read_excel(workbook, r["sheet_idx0"])
else:
@@ -88,45 +90,47 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
return float(passes)
def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
return float(worksheet.freeze_panes==rules["position"])
return float(worksheet.freeze_panes == rules["position"])
if __name__ == '__main__':
#path1 = ""
#path2 = ""
#print(compare_table(path1, path2))
# path1 = ""
# path2 = ""
# print(compare_table(path1, path2))
#path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx"
#path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx"
#print(compare_with_sparklines(path1, path2))
# path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx"
# path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx"
# print(compare_with_sparklines(path1, path2))
#path1 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column_gold.xlsx"
#path2 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column.xlsx"
#workbook1: Workbook = openpyxl.load_workbook(filename=path1)
#worksheet1: Worksheet = workbook1.active
#print(worksheet1.freeze_panes)
#workbook2: Workbook = openpyxl.load_workbook(filename=path2)
#worksheet2: Worksheet = workbook2.active
#print(worksheet2.freeze_panes)
#rule = {"position": "C6"}
#print(check_xlsx_freeze(path1, rule))
# path1 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column_gold.xlsx"
# path2 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column.xlsx"
# workbook1: Workbook = openpyxl.load_workbook(filename=path1)
# worksheet1: Worksheet = workbook1.active
# print(worksheet1.freeze_panes)
# workbook2: Workbook = openpyxl.load_workbook(filename=path2)
# worksheet2: Worksheet = workbook2.active
# print(worksheet2.freeze_panes)
# rule = {"position": "C6"}
# print(check_xlsx_freeze(path1, rule))
#path1 = "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert_gold.xlsx"
#rule = [ { "type": "sheet_name"
#, "sheet_idx": 0
#, "sheet_name": "Sheet1"
#}
#, { "type": "sheet_data"
#, "sheet_idx0": "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert.xlsx@0"
#, "sheet_idx1": 1
#}
#, { "type": "sheet_name"
#, "sheet_idx": 2
#, "sheet_name": "Sheet2"
#}
#]
#print(check_sheet_list(path1, rule))
# path1 = "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert_gold.xlsx"
# rule = [ { "type": "sheet_name"
# , "sheet_idx": 0
# , "sheet_name": "Sheet1"
# }
# , { "type": "sheet_data"
# , "sheet_idx0": "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert.xlsx@0"
# , "sheet_idx1": 1
# }
# , { "type": "sheet_name"
# , "sheet_idx": 2
# , "sheet_name": "Sheet2"
# }
# ]
# print(check_sheet_list(path1, rule))
path1 = "../../../../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold.xlsx"
path2 = "../../../../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold2.xlsx"

18
desktop_env/server/main.py
View File

@@ -3,7 +3,7 @@ from pathlib import Path
import platform
import subprocess
import requests
from .pyxcursor import Xcursor
# import Xlib.display
import pyautogui
# from PIL import ImageGrab, Image
@@ -48,7 +48,7 @@ def capture_screen_with_cursor():
os.makedirs(os.path.dirname(file_path), exist_ok=True)
# fixme: This is a temporary fix for the cursor not being captured on Windows and Linux
if user_platform == "Windows" or user_platform == "Linux":
if user_platform == "Windows":
def _download_image(url, path):
response = requests.get(url)
with open(path, 'wb') as file:
@@ -65,12 +65,14 @@ def capture_screen_with_cursor():
cursor = cursor.resize((int(cursor.width / 1.5), int(cursor.height / 1.5)))
screenshot.paste(cursor, (cursor_x, cursor_y), cursor)
screenshot.save(file_path)
# elif user_platform == "Linux":
# # Use xlib to prevent scrot dependency for Linux
# screen = Xlib.display.Display().screen()
# size = screen.width_in_pixels, screen.height_in_pixels
# screenshot = ImageGrab.grab(bbox=(0, 0, size[0], size[1]))
# screenshot.save(file_path)
elif user_platform == "Linux":
cursor_obj = Xcursor()
imgarray = cursor_obj.getCursorImageArrayFast()
cursor_img = Image.fromarray(imgarray)
screenshot = pyautogui.screenshot()
cursor_x, cursor_y = pyautogui.position()
screenshot.paste(cursor_img, (cursor_x, cursor_y), cursor_img)
screenshot.save(file_path)
elif user_platform == "Darwin": # (Mac OS)
# Use the screencapture utility to capture the screen with the cursor
subprocess.run(["screencapture", "-C", file_path])

146
desktop_env/server/pyxcursor.py Normal file
View File

@@ -0,0 +1,146 @@
import os
import ctypes
import ctypes.util
import numpy as np
# A helper function to convert data from Xlib to byte array.
import struct, array
# Define ctypes version of XFixesCursorImage structure.
PIXEL_DATA_PTR = ctypes.POINTER(ctypes.c_ulong)
Atom = ctypes.c_ulong
class XFixesCursorImage(ctypes.Structure):
"""
See /usr/include/X11/extensions/Xfixes.h
typedef struct {
short x, y;
unsigned short width, height;
unsigned short xhot, yhot;
unsigned long cursor_serial;
unsigned long *pixels;
if XFIXES_MAJOR >= 2
Atom atom; /* Version >= 2 only */
const char *name; /* Version >= 2 only */
endif
} XFixesCursorImage;
"""
_fields_ = [('x', ctypes.c_short),
('y', ctypes.c_short),
('width', ctypes.c_ushort),
('height', ctypes.c_ushort),
('xhot', ctypes.c_ushort),
('yhot', ctypes.c_ushort),
('cursor_serial', ctypes.c_ulong),
('pixels', PIXEL_DATA_PTR),
('atom', Atom),
('name', ctypes.c_char_p)]
class Display(ctypes.Structure):
pass
class Xcursor:
display = None
def __init__(self, display=None):
if not display:
try:
display = os.environ["DISPLAY"].encode("utf-8")
except KeyError:
raise Exception("$DISPLAY not set.")
# XFixeslib = ctypes.CDLL('libXfixes.so')
XFixes = ctypes.util.find_library("Xfixes")
if not XFixes:
raise Exception("No XFixes library found.")
self.XFixeslib = ctypes.cdll.LoadLibrary(XFixes)
# xlib = ctypes.CDLL('libX11.so.6')
x11 = ctypes.util.find_library("X11")
if not x11:
raise Exception("No X11 library found.")
self.xlib = ctypes.cdll.LoadLibrary(x11)
# Define ctypes' version of XFixesGetCursorImage function
XFixesGetCursorImage = self.XFixeslib.XFixesGetCursorImage
XFixesGetCursorImage.restype = ctypes.POINTER(XFixesCursorImage)
XFixesGetCursorImage.argtypes = [ctypes.POINTER(Display)]
self.XFixesGetCursorImage = XFixesGetCursorImage
XOpenDisplay = self.xlib.XOpenDisplay
XOpenDisplay.restype = ctypes.POINTER(Display)
XOpenDisplay.argtypes = [ctypes.c_char_p]
if not self.display:
self.display = self.xlib.XOpenDisplay(display) # (display) or (None)
def argbdata_to_pixdata(self, data, len):
if data == None or len < 1: return None
# Create byte array
b = array.array('b', b'\x00' * 4 * len)
offset, i = 0, 0
while i < len:
argb = data[i] & 0xffffffff
rgba = (argb << 8) | (argb >> 24)
b1 = (rgba >> 24) & 0xff
b2 = (rgba >> 16) & 0xff
b3 = (rgba >> 8) & 0xff
b4 = rgba & 0xff
struct.pack_into("=BBBB", b, offset, b1, b2, b3, b4)
offset = offset + 4
i = i + 1
return b
def getCursorImageData(self):
# Call the function. Read data of cursor/mouse-pointer.
cursor_data = self.XFixesGetCursorImage(self.display)
if not (cursor_data and cursor_data[0]):
raise Exception("Cannot read XFixesGetCursorImage()")
# Note: cursor_data is a pointer, take cursor_data[0]
return cursor_data[0]
def getCursorImageArray(self):
data = self.getCursorImageData()
# x, y = data.x, data.y
height, width = data.height, data.width
bytearr = self.argbdata_to_pixdata(data.pixels, height * width)
imgarray = np.array(bytearr, dtype=np.uint8)
imgarray = imgarray.reshape(height, width, 4)
del bytearr
return imgarray
def getCursorImageArrayFast(self):
data = self.getCursorImageData()
# x, y = data.x, data.y
height, width = data.height, data.width
bytearr = ctypes.cast(data.pixels, ctypes.POINTER(ctypes.c_ulong * height * width))[0]
imgarray = np.array(bytearray(bytearr))
imgarray = imgarray.reshape(height, width, 8)[:, :, (0, 1, 2, 3)]
del bytearr
return imgarray
def saveImage(self, imgarray, text):
from PIL import Image
img = Image.fromarray(imgarray)
img.save(text)
if __name__ == "__main__":
cursor = Xcursor()
imgarray = cursor.getCursorImageArrayFast()
cursor.saveImage(imgarray, 'cursor_image.png')

1
desktop_env/server/requirements.txt
View File

@@ -4,3 +4,4 @@ Pillow==10.1.0
git+https://github.com/moses-palmer/pynput.git@refs/pull/541/head # to make sure that it works on Apple Silicon
requests
flask
numpy

2
main.py
View File

@@ -9,7 +9,7 @@ def human_agent():
with open("evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f:
example = json.load(f)
example["snapshot"] = "base_setup2"
example["snapshot"] = "base_setup3"
env = DesktopEnv(
path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx",