Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -5,6 +5,7 @@ import shutil
|
|
5 |
import os
|
6 |
|
7 |
from customllama3 import CustomLlama3
|
|
|
8 |
|
9 |
from langchain_core.output_parsers import JsonOutputParser
|
10 |
from langchain_core.output_parsers import StrOutputParser
|
@@ -57,23 +58,7 @@ def data_ingestion():
|
|
57 |
st.session_state["console_out"] += "Pdf loaded\n"
|
58 |
except Exception as e:
|
59 |
st.error("PyPDFLoader Exception: " + e)
|
60 |
-
|
61 |
-
docs_list = [item for sublist in docs for item in sublist]
|
62 |
-
|
63 |
-
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
|
64 |
-
chunk_size=250, chunk_overlap=0
|
65 |
-
)
|
66 |
-
doc_splits = text_splitter.split_documents(docs_list)
|
67 |
-
|
68 |
-
embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
|
69 |
-
|
70 |
-
# Add to vectorDB
|
71 |
-
vectorstore = Chroma.from_documents(
|
72 |
-
documents=doc_splits,
|
73 |
-
collection_name="rag-chroma",
|
74 |
-
embedding=embedding_function,
|
75 |
-
)
|
76 |
-
return vectorstore.as_retriever()
|
77 |
|
78 |
def remove_old_files():
|
79 |
st.session_state["console_out"] += "remove_old_files\n"
|
|
|
5 |
import os
|
6 |
|
7 |
from customllama3 import CustomLlama3
|
8 |
+
from ragagent import RAGAgent
|
9 |
|
10 |
from langchain_core.output_parsers import JsonOutputParser
|
11 |
from langchain_core.output_parsers import StrOutputParser
|
|
|
58 |
st.session_state["console_out"] += "Pdf loaded\n"
|
59 |
except Exception as e:
|
60 |
st.error("PyPDFLoader Exception: " + e)
|
61 |
+
return RAGAgent(docs).retriever
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
62 |
|
63 |
def remove_old_files():
|
64 |
st.session_state["console_out"] += "remove_old_files\n"
|