joe4ai commited on
Commit
fba35f8
·
verified ·
1 Parent(s): 80892cd

Update get_data.py

Browse files
Files changed (1) hide show
  1. get_data.py +2 -2
get_data.py CHANGED
@@ -37,11 +37,11 @@ def data_loader(data):
37
  return loader.load()
38
 
39
  def chunk_text(extracted_data):
40
- text_spliter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
41
  text_chunk = text_spliter.split_documents(extracted_data)
42
  return text_chunk
43
 
44
  def download_hugging_face_embeddings():
45
  # Using HuggingFaceEmbeddings from Langchain to load the model
46
- embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
47
  return embeddings
 
37
  return loader.load()
38
 
39
  def chunk_text(extracted_data):
40
+ text_spliter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=100)
41
  text_chunk = text_spliter.split_documents(extracted_data)
42
  return text_chunk
43
 
44
  def download_hugging_face_embeddings():
45
  # Using HuggingFaceEmbeddings from Langchain to load the model
46
+ embeddings=HuggingFaceEmbeddings(model_name='sentence-transformers/multi-qa-MiniLM-L6-cos-v1')
47
  return embeddings