Lhumpal commited on
Commit
0a0e8cf
·
verified ·
1 Parent(s): 182d393

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -5
app.py CHANGED
@@ -22,11 +22,10 @@ google_api_key = os.environ.get("GOOGLE_API_KEY")
22
  login(token=hf_token)
23
 
24
  def chunk_text(text, chunk_size=250, chunk_overlap=0):
25
- # splitter = RecursiveCharacterTextSplitter(
26
- # chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n,\n", "\n", ".", " "]
27
- # )
28
- # chunks = splitter.split_text(text)
29
- chunks = text.split("\n,\n")
30
  return chunks
31
 
32
  # Function to build FAISS index
 
22
  login(token=hf_token)
23
 
24
  def chunk_text(text, chunk_size=250, chunk_overlap=0):
25
+ splitter = RecursiveCharacterTextSplitter(
26
+ chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=[",\n\n", "\n\n", ". "]
27
+ )
28
+ chunks = splitter.split_text(text)
 
29
  return chunks
30
 
31
  # Function to build FAISS index