Spaces:
Runtime error
Runtime error
Delete helper.py
Browse files
helper.py
DELETED
@@ -1,20 +0,0 @@
|
|
1 |
-
from langchain.document_loaders import UnstructuredWordDocumentLoader, DirectoryLoader
|
2 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
3 |
-
from langchain.embeddings import HuggingFaceEmbeddings
|
4 |
-
|
5 |
-
def data_loader(data):
|
6 |
-
loader = DirectoryLoader(
|
7 |
-
data,
|
8 |
-
glob=("*.docx"),
|
9 |
-
loader_cls=UnstructuredWordDocumentLoader,
|
10 |
-
)
|
11 |
-
return loader.load()
|
12 |
-
|
13 |
-
def chunk_text(extracted_data):
|
14 |
-
text_spliter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
|
15 |
-
text_chunk = text_spliter.split_documents(extracted_data)
|
16 |
-
return text_chunk
|
17 |
-
|
18 |
-
def download_hugging_face_embeddings():
|
19 |
-
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
|
20 |
-
return embeddings
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|