Update app.py

app.py

@@ -10,6 +10,8 @@ from langchain.chat_models import ChatOpenAI
 from typing import List
 from together import Together
 from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModel
+import torch
 
 
 import streamlit as st
@@ -107,8 +109,8 @@ from transformers import AutoTokenizer, AutoModel
 
 class HuggingFaceEmbeddings(Embeddings):
     def __init__(self, model_name: str):
-        self.tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-large")
-        self.model = AutoModel.from_pretrained("FacebookAI/xlm-roberta-large")  # Use AutoModel instead of AutoModelForMaskedLM
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModel.from_pretrained(model_name)
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
         embeddings = []
@@ -125,24 +127,27 @@ class HuggingFaceEmbeddings(Embeddings):
 @st.cache_resource
 def get_pdf_index():
     with st.spinner('📄 در حال پردازش فایل PDF...'):
+        # Load the PDF file
         loader = PyPDFLoader('test1.pdf')
         pages = loader.load()
+        
+        # Extract text from each page
         full_text = "\n".join([page.page_content for page in pages])
+        
+        # Split the text into chunks
         text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=1024,  
-            chunk_overlap=128  
+            chunk_size=1024,  # Chunk size
+            chunk_overlap=128  # Overlap between chunks
         )
         texts = text_splitter.split_text(full_text)
         
-        embeddings = HuggingFaceEmbeddings(
-            model_name="FacebookAI/xlm-roberta-large" 
-        )
+        # Create embeddings
+        embeddings = HuggingFaceEmbeddings(model_name="FacebookAI/xlm-roberta-large")
         
-        return VectorstoreIndexCreator(
-            embedding=embeddings,
-            text_splitter=text_splitter
-        ).from_texts(texts)  
-
+        # Create FAISS vector store
+        vector_store = FAISS.from_texts(texts, embeddings)
+        
+        return vector_store
 index = get_pdf_index()
 
 llm = ChatOpenAI(