Update app.py

app.py

@@ -1,6 +1,6 @@
 import streamlit as st
 import pandas as pd
-from transformers import T5Tokenizer, T5ForConditionalGeneration, pipeline
+from transformers import T5Tokenizer, T5ForConditionalGeneration, BertTokenizer, BertModel
 from bertopic import BERTopic
 import torch
 import numpy as np
@@ -9,16 +9,17 @@ import numpy as np
 tokenizer = T5Tokenizer.from_pretrained("UBC-NLP/araT5-base")
 model = T5ForConditionalGeneration.from_pretrained("UBC-NLP/araT5-base")
 
-# Initialize AraBERT model and tokenizer for feature extraction
-bert_tokenizer = pipeline("feature-extraction", model="aubmindlab/bert-base-arabertv2")
+# Initialize BERT tokenizer and model for feature extraction
+bert_tokenizer = BertTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
+bert_model = BertModel.from_pretrained("aubmindlab/bert-base-arabertv2")
 
 # Function to get embeddings from ARAT5 for topic modeling
 def generate_embeddings(texts):
     embeddings = []
-    
+
     for text in texts:
         # Tokenize the text with truncation set to False
-        tokens = bert_tokenizer.tokenizer.encode(text, truncation=False)  # Do not truncate here
+        tokens = bert_tokenizer.encode(text, truncation=False)  # Do not truncate here
 
         # Split the tokens into chunks of size 512 (maximum length)
         chunked_texts = [tokens[i:i + 512] for i in range(0, len(tokens), 512)]
@@ -26,10 +27,11 @@ def generate_embeddings(texts):
         poem_embeddings = []
 
         for chunk in chunked_texts:
-            # Process each chunk and get embeddings
-            inputs = bert_tokenizer.tokenizer(chunk, return_tensors="pt", padding=True, truncation=False, max_length=512)
+            # Prepare the input tensor for the model
+            inputs = torch.tensor(chunk).unsqueeze(0)  # Adding batch dimension
             with torch.no_grad():
-                outputs = bert_tokenizer.model(**inputs)
+                outputs = bert_model(inputs)
+            # Get the embeddings from the last hidden state
             chunk_embedding = outputs.last_hidden_state.mean(dim=1).numpy()
 
             poem_embeddings.append(chunk_embedding)