Update app.py

app.py

@@ -13,11 +13,12 @@ tokenizer = None
 ner_pipeline = None
 pos_pipeline = None
 intent_classifier = None
+semantic_model = None
 models_loaded = False
 
 def load_models(progress=gr.Progress()):
     """Lazy-load models only when needed"""
-    global tokenizer, ner_pipeline, pos_pipeline, intent_classifier, models_loaded
+    global tokenizer, ner_pipeline, pos_pipeline, intent_classifier, semantic_model, models_loaded
     
     if models_loaded:
         return True
@@ -49,6 +50,14 @@ def load_models(progress=gr.Progress()):
             device=0 if torch.cuda.is_available() else -1   # Use GPU if available
         )
         
+        progress(0.9, desc="Loading semantic model...")
+        try:
+            from sentence_transformers import SentenceTransformer
+            semantic_model = SentenceTransformer('all-MiniLM-L6-v2')
+        except Exception as e:
+            print(f"Warning: Could not load semantic model: {str(e)}")
+            semantic_model = None  # Set to None so we can check if it's available
+        
         progress(1.0, desc="Models loaded successfully!")
         models_loaded = True
         return True
@@ -57,6 +66,25 @@ def load_models(progress=gr.Progress()):
         print(f"Error loading models: {str(e)}")
         return f"Error: {str(e)}"
 
+def get_semantic_similarity(token, comparison_terms):
+    """Calculate semantic similarity between a token and comparison terms"""
+    try:
+        from sklearn.metrics.pairwise import cosine_similarity
+        
+        token_embedding = semantic_model.encode([token])[0]
+        comparison_embeddings = semantic_model.encode(comparison_terms)
+        
+        similarities = []
+        for i, emb in enumerate(comparison_embeddings):
+            similarity = cosine_similarity([token_embedding], [emb])[0][0]
+            similarities.append((comparison_terms[i], float(similarity)))
+        
+        return sorted(similarities, key=lambda x: x[1], reverse=True)
+    except Exception as e:
+        print(f"Error in semantic similarity: {str(e)}")
+        # Return dummy data on error
+        return [(term, 0.5) for term in comparison_terms]
+
 def get_token_colors(token_type):
     colors = {
         "prefix": "#D8BFD8",  # Light purple
@@ -272,8 +300,26 @@ def analyze_keyword(keyword, progress=gr.Progress()):
             importance = 60 + (len(token["text"]) * 2)
             importance = min(95, importance)
             
-            # Generate related terms (simplified)
-            related_terms = [f"{token['text']}-related-1", f"{token['text']}-related-2"]
+            # Generate more meaningful related terms using semantic similarity
+            if semantic_model is not None:
+                try:
+                    # Generate some potential related terms
+                    prefix_related = [f"about {token['text']}", f"what is {token['text']}", f"how to {token['text']}"]
+                    synonym_candidates = ["similar", "equivalent", "comparable", "like", "related", "alternative"]
+                    domain_terms = ["software", "marketing", "business", "science", "education", "technology"]
+                    comparison_terms = prefix_related + synonym_candidates + domain_terms
+                    
+                    # Get similarities
+                    similarities = get_semantic_similarity(token['text'], comparison_terms)
+                    
+                    # Use top 3 most similar terms
+                    related_terms = [term for term, score in similarities[:3]]
+                except Exception as e:
+                    print(f"Error generating semantic related terms: {str(e)}")
+                    related_terms = [f"{token['text']}-related-1", f"{token['text']}-related-2"]
+            else:
+                # Fallback if semantic model isn't loaded
+                related_terms = [f"{token['text']}-related-1", f"{token['text']}-related-2"]
             
             full_token_analysis.append({
                 "token": token["text"],