diff --git a/.gitignore b/.gitignore index 5d381cc..416ae75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,9 @@ # ---> Python + +# model +*.pt +/home/ubuntu/sas0/LYT/mars1215/mars_toolkit/model/eqV2_86M_omat_mp_salex.pt + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/chart_screenshot.png b/chart_screenshot.png new file mode 100644 index 0000000..1dede93 Binary files /dev/null and b/chart_screenshot.png differ diff --git a/constant.py b/constant.py new file mode 100644 index 0000000..a36a7fe --- /dev/null +++ b/constant.py @@ -0,0 +1,10 @@ +TOPK_RESULT = 5 +TIME_OUT = 30 +MP_API_KEY = "gfBp2in8qxm9Xm2SwLKFwNxDyZvNTAEt" + +# MinIO configuration +MINIO_ENDPOINT = "https://s3-api.siat-mic.com" +INTERNEL_MINIO_ENDPOINT = "http://100.85.52.31:9000" # 内网地址,如果有就填,上传会更快。 +MINIO_ACCESS_KEY = "9bUtQL1Gpo9JB6o3pSGr" +MINIO_SECRET_KEY = "1Qug5H73R3kP8boIHvdVcFtcb1jU9GRWnlmMpx0g" +MINIO_BUCKET = "temp" diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database/material_project_router.py b/database/material_project_router.py new file mode 100644 index 0000000..aae911c --- /dev/null +++ b/database/material_project_router.py @@ -0,0 +1,163 @@ +""" +Author: Yutang LI +Institution: SIAT-MIC +Contact: yt.li2@siat.ac.cn +""" + +from fastapi import APIRouter, Request +import json +import asyncio +import logging +from mp_api.client import MPRester +from multiprocessing import Process, Manager +from typing import Dict, Any, List +from constant import MP_API_KEY, TIME_OUT, TOPK_RESULT + +router = APIRouter(prefix="/mp", tags=["Material Project"]) + +logger = logging.getLogger(__name__) + + +@router.get("/search") +async def search_from_material_project(request: Request): + # 打印请求日志 + logger.info(f"Received request: {request.method} {request.url}") + logger.info(f"Query parameters: {request.query_params}") + + # 解析查询参数 + search_args = parse_search_parameters(request.query_params) + + # 检查API key + if MP_API_KEY is None or MP_API_KEY == '': + return 'Material Project API CANNOT Be None' + + try: + # 执行搜索 + docs = await execute_search(search_args) + + # 处理搜索结果 + res = process_search_results(docs) + + # 返回结果 + if len(res) >= TOPK_RESULT: + return json.dumps(res[:TOPK_RESULT], indent=2) + return json.dumps(res, indent=2) + + except asyncio.TimeoutError: + logger.error(f"Request timed out after {TIME_OUT} seconds, please simplify your query and try again.") + return {"error": f"Request timed out after {TIME_OUT} seconds, please simplify your query and try again."} + + +def parse_bool(param: str) -> bool | None: + if not param: + return None + return param.lower() == 'true' + + +def parse_list(param: str) -> List[str] | None: + if not param: + return None + return param.split(',') + + +def parse_tuple(param: str) -> tuple[float, float] | None: + if not param: + return None + try: + values = param.split(',') + return (float(values[0]), float(values[1])) + except (ValueError, IndexError): + return None + + +def parse_search_parameters(query_params: Dict[str, str]) -> Dict[str, Any]: + """解析搜索参数""" + return { + 'band_gap': parse_tuple(query_params.get('band_gap')), + 'chemsys': parse_list(query_params.get('chemsys')), + 'crystal_system': parse_list(query_params.get('crystal_system')), + 'density': parse_tuple(query_params.get('density')), + 'formation_energy': parse_tuple(query_params.get('formation_energy')), + 'elements': parse_list(query_params.get('elements')), + 'exclude_elements': parse_list(query_params.get('exclude_elements')), + 'formula': parse_list(query_params.get('formula')), + 'is_gap_direct': parse_bool(query_params.get('is_gap_direct')), + 'is_metal': parse_bool(query_params.get('is_metal')), + 'is_stable': parse_bool(query_params.get('is_stable')), + 'magnetic_ordering': query_params.get('magnetic_ordering'), + 'material_ids': parse_list(query_params.get('material_ids')), + 'total_energy': parse_tuple(query_params.get('total_energy')), + 'num_elements': parse_tuple(query_params.get('num_elements')), + 'volume': parse_tuple(query_params.get('volume')), + 'chunk_size': int(query_params.get('chunk_size', '5')) + } + + +def process_search_results(docs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """处理搜索结果""" + fields = [ + 'formula_pretty', 'nsites', 'nelements', 'material_id', 'chemsys', + 'volume', 'density', 'density_atomic', 'cbm', 'vbm', 'band_gap', + 'is_gap_direct', 'is_stable', 'formation_energy_per_atom', + 'energy_above_hull', 'is_metal', 'total_magnetization', 'efermi', + 'is_magnetic', 'ordering', 'bulk_modulus', 'shear_modulus', + 'universal_anisotropy', 'theoretical' + ] + + res = [] + for doc in docs: + try: + new_docs = {} + for field_name in fields: + new_docs[field_name] = doc.get(field_name, '') + res.append(new_docs) + except Exception as e: + logger.warning(f"Error processing document: {str(e)}") + continue + return res + + +async def execute_search(search_args: Dict[str, Any], timeout: int = TIME_OUT) -> List[Dict[str, Any]]: + """执行搜索""" + manager = Manager() + queue = manager.Queue() + + p = Process(target=_search_worker, args=(queue, MP_API_KEY), kwargs=search_args) + p.start() + + logger.info(f"Started worker process with PID: {p.pid}") + p.join(timeout=timeout) + + if p.is_alive(): + logger.warning(f"Terminating worker process {p.pid} due to timeout") + p.terminate() + p.join() + raise asyncio.TimeoutError(f"Request timed out after {timeout} seconds") + + try: + if queue.empty(): + logger.warning("Queue is empty after process completion") + else: + logger.info("Queue contains data, retrieving...") + + result = queue.get(timeout=15) + except queue.Empty: + logger.error("Failed to retrieve data from queue") + raise RuntimeError("Failed to retrieve data from worker process") + + if isinstance(result, Exception): + logger.error(f"Error in search worker: {str(result)}") + raise result + + logger.info(f"Successfully retrieved {len(result)} documents") + return result + + +def _search_worker(queue, api_key, **kwargs): + """搜索工作线程""" + try: + mpr = MPRester(api_key) + result = mpr.materials.summary.search(**kwargs) + queue.put([doc.model_dump() for doc in result]) + except Exception as e: + queue.put(e) diff --git a/database/oqmd_router.py b/database/oqmd_router.py new file mode 100644 index 0000000..c5153c2 --- /dev/null +++ b/database/oqmd_router.py @@ -0,0 +1,305 @@ +""" +Author: Yutang LI +Institution: SIAT-MIC +Contact: yt.li2@siat.ac.cn +""" +import os +import boto3 +from fastapi import APIRouter, Request +from io import StringIO +import logging +import httpx +import datetime +import pandas as pd +from bs4 import BeautifulSoup +from PIL import Image +from playwright.async_api import async_playwright +from constant import MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_BUCKET, INTERNEL_MINIO_ENDPOINT + + +router = APIRouter(prefix="/oqmd", tags=["OQMD"]) +logger = logging.getLogger(__name__) + + +@router.get("/search") +async def search_from_oqmd_by_composition(request: Request): + # 打印请求日志 + logger.info(f"Received request: {request.method} {request.url}") + logger.info(f"Query parameters: {request.query_params}") + + try: + # 获取并解析数据 + composition = request.query_params['composition'] + html = await fetch_oqmd_data(composition) + basic_data, table_data, phase_data = parse_oqmd_html(html) + + # 渲染并保存图表 + phase_diagram_name = await render_and_save_charts(phase_data) + # 返回格式化后的响应 + return format_response(basic_data, table_data, phase_diagram_name) + + except httpx.HTTPStatusError as e: + logger.error(f"OQMD API request failed: {str(e)}") + return { + "status": "error", + "message": f"OQMD API request failed: {str(e)}" + } + except Exception as e: + logger.error(f"Unexpected error: {str(e)}") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } + + + +async def fetch_oqmd_data(composition: str) -> str: + """ + 从OQMD获取数据 + Args: + composition: 材料组成字符串 + + Returns: + HTML内容字符串 + + Raises: + httpx.HTTPError: 当发生HTTP相关错误时抛出 + ValueError: 当响应内容无效时抛出 + """ + url = f"https://www.oqmd.org/materials/composition/{composition}" + try: + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(url) + response.raise_for_status() + + # 验证响应内容 + if not response.text or len(response.text) < 100: + raise ValueError("Invalid response content from OQMD API") + + return response.text + + except httpx.HTTPStatusError as e: + status_code = e.response.status_code + if status_code == 401: + logger.error("OQMD API: Unauthorized access") + raise httpx.HTTPError("Unauthorized access to OQMD API") from e + elif status_code == 403: + logger.error("OQMD API: Forbidden access") + raise httpx.HTTPError("Forbidden access to OQMD API") from e + elif status_code == 404: + logger.error("OQMD API: Resource not found") + raise httpx.HTTPError("Resource not found on OQMD API") from e + elif status_code >= 500: + logger.error(f"OQMD API: Server error ({status_code})") + raise httpx.HTTPError(f"OQMD API server error ({status_code})") from e + else: + logger.error(f"OQMD API request failed: {str(e)}") + raise httpx.HTTPError(f"OQMD API request failed: {str(e)}") from e + + except httpx.TimeoutException as e: + logger.error("OQMD API request timed out") + raise httpx.HTTPError("OQMD API request timed out") from e + + except httpx.NetworkError as e: + logger.error(f"Network error occurred: {str(e)}") + raise httpx.HTTPError(f"Network error: {str(e)}") from e + + except ValueError as e: + logger.error(f"Invalid response content: {str(e)}") + raise ValueError(f"Invalid response content: {str(e)}") from e + +def parse_oqmd_html(html: str) -> tuple[list, str, list]: + """ + 解析OQMD HTML数据 + """ + soup = BeautifulSoup(html, 'html.parser') + # 解析基本数据 + basic_data = [] + basic_data.append(soup.find('h1').text.strip()) + for script in soup.find_all('p'): + if script: + combined_text = "" + for element in script.contents: # 遍历