Update app.py
Browse files
app.py
CHANGED
@@ -22,11 +22,10 @@ google_api_key = os.environ.get("GOOGLE_API_KEY")
|
|
22 |
login(token=hf_token)
|
23 |
|
24 |
def chunk_text(text, chunk_size=250, chunk_overlap=0):
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
chunks = text.split("\n,\n")
|
30 |
return chunks
|
31 |
|
32 |
# Function to build FAISS index
|
|
|
22 |
login(token=hf_token)
|
23 |
|
24 |
def chunk_text(text, chunk_size=250, chunk_overlap=0):
|
25 |
+
splitter = RecursiveCharacterTextSplitter(
|
26 |
+
chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=[",\n\n", "\n\n", ". "]
|
27 |
+
)
|
28 |
+
chunks = splitter.split_text(text)
|
|
|
29 |
return chunks
|
30 |
|
31 |
# Function to build FAISS index
|