28 lines
1.3 KiB
Python
28 lines
1.3 KiB
Python
|
from llama_index.core import SimpleDirectoryReader
|
|||
|
|
from llama_index.readers.dashscope.base import DashScopeParse
|
|||
|
|
from llama_index.readers.dashscope.utils import ResultType
|
|||
|
|
from llama_index.indices.managed.dashscope import DashScopeCloudIndex
|
|||
|
|
|
|||
|
|
|
|||
|
|
def read_parse_upload_local_documents(dir, num_workers=1):
|
|||
|
|
"""读取、解析、上传本地文件到百炼数据管理平台。
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
dir (str): 本地文件存储的路径。
|
|||
|
|
num_workers (int, optional): 执行的并发数。
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
已上传到云端的文件列表
|
|||
|
|
"""
|
|||
|
|
parse = DashScopeParse(result_type=ResultType.DASHSCOPE_DOCMIND)
|
|||
|
|
file_extractor = {'.txt': parse, '.docx': parse, ".pdf": parse} # 设置需要读取解析的文件格式，请根据实际需求调整
|
|||
|
|
documents = SimpleDirectoryReader(input_dir=dir, file_extractor=file_extractor).load_data(num_workers=num_workers)
|
|||
|
|
|
|||
|
|
return documents
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
dir = "./docs/" # 本例中，业务相关文件存储在当前路径下的docs文件夹，请根据实际情况调整。
|
|||
|
|
documents = read_parse_upload_local_documents(dir)
|
|||
|
|
cloud_index_name = "my_first_index" # 设置云端知识库索引名称
|
|||
|
|
index = DashScopeCloudIndex.from_documents(documents, cloud_index_name, verbose=True) # 创建云端知识库索引
|