46 lines
1.4 KiB
Python
46 lines
1.4 KiB
Python
"""
|
|
0. 将问题从xls提取为json
|
|
1. 将问题进行拆分
|
|
2. 翻译成英文
|
|
3. 去重
|
|
4. 使用大模型进行难度评估和筛选
|
|
"""
|
|
import pandas as pd
|
|
import json
|
|
import os
|
|
|
|
def process_excel_files(directory):
|
|
all_data = []
|
|
|
|
# 获取目录下所有xlsx文件
|
|
excel_files = [f for f in os.listdir(directory) if f.endswith('.xlsx')]
|
|
|
|
for excel_file in excel_files:
|
|
|
|
file_path = os.path.join(directory, excel_file)
|
|
|
|
df = pd.read_excel(file_path)
|
|
|
|
if 'Question' in df.columns and 'Answer' in df.columns:
|
|
# 将每行转换为字典并添加到列表中
|
|
for _, row in df.iterrows():
|
|
data_item = {
|
|
'question': str(row['Question']).strip(),
|
|
'answer': str(row['Answer']).strip()
|
|
}
|
|
all_data.append(data_item)
|
|
else:
|
|
print(f"警告: {excel_file} 缺少必要的列 (question/answer)")
|
|
|
|
# 将数据保存为JSON文件
|
|
output_file = os.path.join(directory, 'qa_data.json')
|
|
with open(output_file, 'w', encoding='utf-8') as f:
|
|
json.dump(all_data, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"处理完成！共处理了 {len(all_data)} 条数据")
|
|
print(f"数据已保存到: {output_file}")
|
|
|
|
if __name__ == '__main__':
|
|
# 指定Excel文件所在的目录
|
|
directory = os.path.dirname(os.path.abspath(__file__))
|
|
process_excel_files(directory) |