424 lines
17 KiB
Python
424 lines
17 KiB
Python
import json
|
||
import asyncio
|
||
import concurrent.futures
|
||
|
||
import jsonlines
|
||
from mars_toolkit import *
|
||
import threading
|
||
import uuid
|
||
|
||
from mars_toolkit.compute.material_gen import generate_material
|
||
# Create a lock for file writing
|
||
file_lock = threading.Lock()
|
||
from mysql.connector import pooling
|
||
from colorama import Fore, Back, Style, init
|
||
import time
|
||
import random
|
||
# 初始化colorama
|
||
init(autoreset=True)
|
||
|
||
from typing import Dict, Union, Any, Optional
|
||
|
||
def normalize_material_args(arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""
|
||
规范化传递给generate_material函数的参数格式。
|
||
|
||
处理以下情况：
|
||
1. properties参数可能是字符串形式的JSON，需要解析为字典
|
||
2. properties中的值可能需要转换为适当的类型（数字或字符串）
|
||
3. 确保batch_size和num_batches是整数
|
||
|
||
Args:
|
||
arguments: 包含generate_material参数的字典
|
||
|
||
Returns:
|
||
规范化后的参数字典
|
||
"""
|
||
normalized_args = arguments.copy()
|
||
|
||
# 处理properties参数
|
||
if "properties" in normalized_args:
|
||
properties = normalized_args["properties"]
|
||
|
||
# 如果properties是字符串，尝试解析为JSON
|
||
if isinstance(properties, str):
|
||
try:
|
||
properties = json.loads(properties)
|
||
except json.JSONDecodeError as e:
|
||
raise ValueError(f"无法解析properties JSON字符串: {e}")
|
||
|
||
# 确保properties是字典
|
||
if not isinstance(properties, dict):
|
||
raise ValueError(f"properties必须是字典或JSON字符串，而不是 {type(properties)}")
|
||
|
||
# 处理properties中的值
|
||
normalized_properties = {}
|
||
for key, value in properties.items():
|
||
# 处理范围值，例如 "0.0-2.0" 或 "40-50"
|
||
if isinstance(value, str) and "-" in value and not value.startswith(">") and not value.startswith("<"):
|
||
# 保持范围值为字符串格式
|
||
normalized_properties[key] = value
|
||
elif isinstance(value, str) and value.startswith(">"):
|
||
# 保持大于值为字符串格式
|
||
normalized_properties[key] = value
|
||
elif isinstance(value, str) and value.startswith("<"):
|
||
# 保持小于值为字符串格式
|
||
normalized_properties[key] = value
|
||
elif isinstance(value, str) and value.lower() == "relaxor":
|
||
# 特殊值保持为字符串
|
||
normalized_properties[key] = value
|
||
elif isinstance(value, str) and value.endswith("eV"):
|
||
# 带单位的值保持为字符串
|
||
normalized_properties[key] = value
|
||
else:
|
||
# 尝试将值转换为数字
|
||
try:
|
||
# 如果可以转换为浮点数
|
||
float_value = float(value)
|
||
# 如果是整数，转换为整数
|
||
if float_value.is_integer():
|
||
normalized_properties[key] = int(float_value)
|
||
else:
|
||
normalized_properties[key] = float_value
|
||
except (ValueError, TypeError):
|
||
# 如果无法转换为数字，保持原值
|
||
normalized_properties[key] = value
|
||
|
||
normalized_args["properties"] = normalized_properties
|
||
|
||
# 确保batch_size和num_batches是整数
|
||
if "batch_size" in normalized_args:
|
||
try:
|
||
normalized_args["batch_size"] = int(normalized_args["batch_size"])
|
||
except (ValueError, TypeError):
|
||
raise ValueError(f"batch_size必须是整数，而不是 {normalized_args['batch_size']}")
|
||
|
||
if "num_batches" in normalized_args:
|
||
try:
|
||
normalized_args["num_batches"] = int(normalized_args["num_batches"])
|
||
except (ValueError, TypeError):
|
||
raise ValueError(f"num_batches必须是整数，而不是 {normalized_args['num_batches']}")
|
||
|
||
# 确保diffusion_guidance_factor是浮点数
|
||
if "diffusion_guidance_factor" in normalized_args:
|
||
try:
|
||
normalized_args["diffusion_guidance_factor"] = float(normalized_args["diffusion_guidance_factor"])
|
||
except (ValueError, TypeError):
|
||
raise ValueError(f"diffusion_guidance_factor必须是数字，而不是 {normalized_args['diffusion_guidance_factor']}")
|
||
|
||
return normalized_args
|
||
|
||
|
||
import requests
|
||
connection_pool = pooling.MySQLConnectionPool(
|
||
pool_name="mypool",
|
||
pool_size=32,
|
||
pool_reset_session=True,
|
||
host='localhost',
|
||
user='metadata_mat_papers',
|
||
password='siat-mic',
|
||
database='metadata_mat_papers'
|
||
)
|
||
|
||
async def process_retrieval_from_knowledge_base(data):
|
||
doi = data.get('doi')
|
||
mp_id = data.get('mp_id')
|
||
|
||
# 检查是否提供了至少一个查询参数
|
||
if doi is None and mp_id is None:
|
||
return "" # 如果没有提供查询参数，返回空字符串
|
||
|
||
# 构建SQL查询条件
|
||
query = "SELECT * FROM mp_synthesis_scheme_info WHERE "
|
||
params = []
|
||
|
||
if doi is not None and mp_id is not None:
|
||
query += "doi = %s OR mp_id = %s"
|
||
params = [doi, mp_id]
|
||
elif doi is not None:
|
||
query += "doi = %s"
|
||
params = [doi]
|
||
else: # mp_id is not None
|
||
query += "mp_id = %s"
|
||
params = [mp_id]
|
||
|
||
# 从数据库中查询匹配的记录
|
||
conn = connection_pool.get_connection()
|
||
try:
|
||
cursor = conn.cursor(dictionary=True)
|
||
try:
|
||
cursor.execute(query, params)
|
||
result = cursor.fetchone() # 获取第一个匹配的记录
|
||
finally:
|
||
cursor.close()
|
||
finally:
|
||
conn.close()
|
||
|
||
# 检查是否找到匹配的记录
|
||
if not result:
|
||
return "" # 如果没有找到匹配记录，返回空字符串
|
||
|
||
# 构建markdown格式的结果
|
||
markdown_result = ""
|
||
|
||
# 添加各个字段（除了doi和mp_id）
|
||
fields = [
|
||
"target_material",
|
||
"reaction_string",
|
||
"chara_structure",
|
||
"chara_performance",
|
||
"chara_application",
|
||
"synthesis_schemes"
|
||
]
|
||
|
||
for field in fields:
|
||
# 获取字段内容
|
||
field_content = result.get(field, "")
|
||
# 只有当字段内容不为空时才添加该字段
|
||
if field_content and field_content.strip():
|
||
markdown_result += f"\n## {field}\n{field_content}\n\n"
|
||
|
||
return markdown_result # 直接返回markdown文本
|
||
|
||
|
||
|
||
async def execute_tool_from_dict(input_dict: dict):
|
||
"""
|
||
从字典中提取工具函数名称和参数，并执行相应的工具函数
|
||
|
||
Args:
|
||
input_dict: 字典，例如：
|
||
{"name": "search_material_property_from_material_project",
|
||
"arguments": "{\"formula\": \"Th3Pd5\", \"is_stable\": \"true\"}"}
|
||
|
||
Returns:
|
||
工具函数的执行结果，如果工具函数不存在则返回错误信息
|
||
"""
|
||
try:
|
||
# 解析输入字符串为字典
|
||
# input_dict = json.loads(input_str)
|
||
|
||
# 提取函数名和参数
|
||
func_name = input_dict.get("name")
|
||
arguments_data = input_dict.get("arguments")
|
||
#print('func_name', func_name)
|
||
#print("argument", arguments_data)
|
||
if not func_name:
|
||
return {"status": "error", "message": "未提供函数名称"}
|
||
|
||
# 获取所有注册的工具函数
|
||
tools = get_tools()
|
||
|
||
# 检查函数名是否存在于工具函数字典中
|
||
if func_name not in tools:
|
||
return {"status": "error", "message": f"函数 '{func_name}' 不存在于工具函数字典中"}
|
||
|
||
# 获取对应的工具函数
|
||
tool_func = tools[func_name]
|
||
|
||
# 处理参数
|
||
arguments = {}
|
||
if arguments_data:
|
||
# 检查arguments是字符串还是字典
|
||
if isinstance(arguments_data, dict):
|
||
# 如果已经是字典，直接使用
|
||
arguments = arguments_data
|
||
elif isinstance(arguments_data, str):
|
||
# 如果是字符串，尝试解析为JSON
|
||
try:
|
||
# 尝试直接解析为JSON对象
|
||
arguments = json.loads(arguments_data)
|
||
except json.JSONDecodeError:
|
||
# 如果解析失败，可能是因为字符串中包含转义字符
|
||
# 尝试修复常见的JSON字符串问题
|
||
fixed_str = arguments_data.replace('\\"', '"').replace('\\\\', '\\')
|
||
try:
|
||
arguments = json.loads(fixed_str)
|
||
except json.JSONDecodeError:
|
||
# 如果仍然失败，尝试将字符串作为原始字符串处理
|
||
arguments = {"raw_string": arguments_data}
|
||
|
||
# 调用工具函数
|
||
if asyncio.iscoroutinefunction(tool_func):
|
||
# 如果是异步函数，使用await调用
|
||
result = await tool_func(**arguments)
|
||
else:
|
||
# 如果是同步函数，直接调用
|
||
result = tool_func(**arguments)
|
||
# if func_name=='generate_material':
|
||
# print("xxxxx",result)
|
||
return result
|
||
|
||
except json.JSONDecodeError as e:
|
||
return {"status": "error", "message": f"JSON解析错误: {str(e)}"}
|
||
except Exception as e:
|
||
return {"status": "error", "message": f"执行过程中出错: {str(e)}"}
|
||
|
||
|
||
def worker(data, output_file_path):
|
||
try:
|
||
func_contents = data["function_calls"]
|
||
func_results = []
|
||
formatted_results = [] # 新增一个列表来存储格式化后的结果
|
||
for func in func_contents:
|
||
func_name = func.get("name")
|
||
arguments_data = func.get("arguments")
|
||
|
||
# 使用富文本打印函数名
|
||
print(f"{Fore.CYAN}{Style.BRIGHT}【函数名】{Style.RESET_ALL} {Fore.YELLOW}{func_name}{Style.RESET_ALL}")
|
||
|
||
# 使用富文本打印参数
|
||
print(f"{Fore.CYAN}{Style.BRIGHT}【参数】{Style.RESET_ALL} {Fore.GREEN}{arguments_data}{Style.RESET_ALL}")
|
||
|
||
if func.get("name") == 'retrieval_from_knowledge_base':
|
||
pass
|
||
# delay_time = random.uniform(5, 10)
|
||
# time.sleep(delay_time)
|
||
result = asyncio.run(process_retrieval_from_knowledge_base(data))
|
||
func_results.append({"function": func['name'], "result": result})
|
||
# 格式化结果
|
||
formatted_result = f"[{func_name} content begin]{result}[{func_name} content end]"
|
||
formatted_results.append(formatted_result)
|
||
|
||
elif func.get("name") == 'generate_material':
|
||
# # 规范化参数
|
||
# try:
|
||
# # 确保arguments_data是字典
|
||
# if isinstance(arguments_data, str):
|
||
# try:
|
||
# arguments_data = json.loads(arguments_data)
|
||
# except json.JSONDecodeError as e:
|
||
# print(f"{Fore.RED}无法解析arguments_data JSON字符串: {e}{Style.RESET_ALL}")
|
||
# continue
|
||
|
||
# # 规范化参数
|
||
# normalized_args = normalize_material_args(arguments_data)
|
||
# # print(f"{Fore.CYAN}{Style.BRIGHT}【函数名】{Style.RESET_ALL} {Fore.YELLOW}{func_name}{Style.RESET_ALL}")
|
||
# # print(f"{Fore.CYAN}{Style.BRIGHT}【原始参数】{Style.RESET_ALL} {Fore.GREEN}{json.dumps(arguments_data, ensure_ascii=False, indent=2)}{Style.RESET_ALL}")
|
||
# # print(f"{Fore.CYAN}{Style.BRIGHT}【规范化参数】{Style.RESET_ALL} {Fore.GREEN}{json.dumps(normalized_args, ensure_ascii=False, indent=2)}{Style.RESET_ALL}")
|
||
|
||
# # 优先使用mattergen函数
|
||
# try:
|
||
# # output = asyncio.run(generate_material(**normalized_args))
|
||
# output = generate_material(**normalized_args)
|
||
|
||
# # 添加延迟，模拟额外的工具函数调用
|
||
|
||
# # 随机延迟5-10秒
|
||
# # delay_time = random.uniform(5, 10)
|
||
# # print(f"{Fore.MAGENTA}{Style.BRIGHT}正在执行额外的工具函数调用，预计需要 {delay_time:.2f} 秒...{Style.RESET_ALL}")
|
||
# # time.sleep(delay_time)
|
||
|
||
# # # 模拟其他工具函数调用的日志输出
|
||
# # print(f"{Fore.BLUE}正在分析生成的材料结构...{Style.RESET_ALL}")
|
||
# # time.sleep(0.5)
|
||
# # print(f"{Fore.BLUE}正在计算结构稳定性...{Style.RESET_ALL}")
|
||
# # time.sleep(0.5)
|
||
# # print(f"{Fore.BLUE}正在验证属性约束条件...{Style.RESET_ALL}")
|
||
# # time.sleep(0.5)
|
||
# # print(f"{Fore.GREEN}{Style.BRIGHT}额外的工具函数调用完成{Style.RESET_ALL}")
|
||
|
||
# except Exception as e:
|
||
# print(f"{Fore.RED}mattergen出错，尝试使用generate_material: {str(e)}{Style.RESET_ALL}")
|
||
|
||
# # 将结果添加到func_results
|
||
# func_results.append({"function": func_name, "result": output})
|
||
|
||
# # 格式化结果
|
||
# formatted_result = f"[{func_name} content begin]{output}[{func_name} content end]"
|
||
# formatted_results.append(formatted_result)
|
||
# except Exception as e:
|
||
# print(f"{Fore.RED}处理generate_material参数时出错: {e}{Style.RESET_ALL}")
|
||
# import traceback
|
||
# print(f"{Fore.RED}{traceback.format_exc()}{Style.RESET_ALL}")
|
||
pass
|
||
else:
|
||
# delay_time = random.uniform(5, 10)
|
||
# time.sleep(delay_time)
|
||
|
||
result = asyncio.run(execute_tool_from_dict(func))
|
||
func_results.append({"function": func['name'], "result": result})
|
||
# 格式化结果
|
||
func_name = func.get("name")
|
||
formatted_result = f"[{func_name} content begin]{result}[{func_name} content end]"
|
||
formatted_results.append(formatted_result)
|
||
|
||
# 将所有格式化后的结果连接起来
|
||
final_result = "\n\n\n".join(formatted_results)
|
||
data['observation'] = final_result
|
||
|
||
# 使用富文本打印开始和结束标记
|
||
print(f"{Back.BLUE}{Fore.WHITE}{Style.BRIGHT}{'#'*50} 结果开始 {'#'*50}{Style.RESET_ALL}")
|
||
print(data['observation'])
|
||
print(f"{Back.BLUE}{Fore.WHITE}{Style.BRIGHT}{'#'*50} 结果结束 {'#'*50}{Style.RESET_ALL}")
|
||
with file_lock:
|
||
with jsonlines.open(output_file_path, mode='a') as writer:
|
||
writer.write(data) # observation . data
|
||
return f"Processed successfully"
|
||
|
||
except Exception as e:
|
||
print(f"{Fore.RED}{Style.BRIGHT}处理过程中出错: {str(e)}{Style.RESET_ALL}")
|
||
return f"Error processing: {str(e)}"
|
||
|
||
|
||
def main(datas, output_file_path, max_workers=1):
|
||
import random
|
||
from tqdm import tqdm
|
||
import os
|
||
from mysql.connector import pooling, Error
|
||
|
||
# 创建进度条
|
||
pbar = tqdm(total=len(datas), desc="Processing papers")
|
||
|
||
# 创建一个线程池
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||
# 提交任务到执行器
|
||
future_to_path = {}
|
||
for path in datas:
|
||
future = executor.submit(worker, path, output_file_path)
|
||
future_to_path[future] = path
|
||
|
||
# 处理结果
|
||
completed = 0
|
||
failed = 0
|
||
for future in concurrent.futures.as_completed(future_to_path):
|
||
path = future_to_path[future]
|
||
try:
|
||
result = future.result()
|
||
if "successfully" in result:
|
||
completed += 1
|
||
else:
|
||
failed += 1
|
||
# 更新进度条
|
||
pbar.update(1)
|
||
# 每100个文件更新一次统计信息
|
||
if (completed + failed) % 100 == 0:
|
||
pbar.set_postfix(completed=completed, failed=failed)
|
||
except Exception as e:
|
||
failed += 1
|
||
pbar.update(1)
|
||
print(f"\nWorker for {path} generated an exception: {e}")
|
||
|
||
pbar.close()
|
||
print(f"Processing complete. Successfully processed: {completed}, Failed: {failed}")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
import datetime
|
||
import jsonlines
|
||
datas = []
|
||
with jsonlines.open('/home/ubuntu/sas0/LYT/mars1215/make_reason_src/filter_failed_questions_solutions_20250323140107.jsonl') as reader:
|
||
for obj in reader:
|
||
datas.append(obj)
|
||
|
||
print(len(datas))
|
||
# print()
|
||
output_file = f"./filter_ok_questions_solutions_agent_other_tools_{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}.jsonl"
|
||
main(datas, output_file, max_workers=32)
|
||
|
||
# 示例1：使用正确的JSON格式
|
||
# argument = '{"properties": {"chemical_system": "V-Zn-O", "crystal_system": "monoclinic", "space_group": "P21/c", "volume": 207.37}, "batch_size": 1, "num_batches": 1}'
|
||
# argument = json.loads(argument)
|
||
# print(json.dumps(argument, indent=2))
|
||
# asyncio.run(mattergen(**argument))
|