Langchain支持SKLearnVectorStore存储

scikit-learn是开源的机器学习算法python库，尤其那个K-近邻算法(K-Nearest Neighbors)更是“家喻户晓”,现在langchain的向量存储也支持SKLearnVectorStore，下面我们来看看怎么使用

1.安装环境

pip install scikit-learn #sk算法库

pip install pandas pyarrow #需要对parquet格式支持

2.如何使用sk-vectorstore

import os

from langchain.embeddings import HuggingFaceEmbeddings

from langchain.text_splitter import CharacterTextSplitter

#引入SKLearnVectorStore

from langchain.vectorstores import SKLearnVectorStore

from langchain.document_loaders import TextLoader

##文件切割

loader = TextLoader('./data/data_c.txt')

documents = loader.load()

text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)

docs = text_splitter.split_documents(documents)

##向量化model

embeddings = HuggingFaceEmbeddings()

import tempfile

persist_path = os.path.join(tempfile.gettempdir(), 'data_c.parquet')

#创建SKLearnVectorStore

vector_store = SKLearnVectorStore.from_documents(

documents=docs,

embedding=embeddings,

persist_path=persist_path,#持久化目录

serializer='parquet' #存储格式parquet

)

#查询

query = "What did the president say about Ketanji Brown Jackson"

docs = vector_store.similarity_search(query)

print(docs[0].page_content)

##持久化

vector_store.persist()

print('Vector store was persisted to', persist_path)

#加载之前持久化的数据

vector_store_load = SKLearnVectorStore(

embedding=embeddings,

persist_path=persist_path,

serializer='parquet'

)

print('A new instance of vector store was loaded from', persist_path)

#查询

docs = vector_store_load.similarity_search(query)

print(docs[0].page_content)

#清理数据

os.remove(persist_path)