Update app.py

app.py

@@ -5,10 +5,13 @@ import gradio as gr
 import time
 import os
 import json
-import aiohttp
-import aiofiles
 import re
 from datetime import datetime
+import torch
+from transformers import AutoTokenizer, AutoModel
+import networkx as nx
+from pyvis.network import Network
+import matplotlib.pyplot as plt
 
 # 🧙‍♂️ Magical Utility Functions 🧙‍♂️
 
@@ -16,20 +19,6 @@ def safe_filename(title):
     """Convert a string to a safe filename. No more 'file not found' nightmares! 🙅‍♂️📁"""
     return re.sub(r'[^\w\-_\. ]', '_', title)
 
-# 🎬 Animated Banner Messages 🎬
-def animated_banner(message, emoji):
-    """Create an animated banner message. It's like a tiny parade for your console! 🎉🚩"""
-    frames = [
-        f"╔════ {emoji} ════╗\n║ {message:^16} ║\n╚════════════╝",
-        f"╔════ {emoji} ════╗\n║ {message:^16} ║\n╚════════════╝",
-        f"╔════{emoji}════╗\n║ {message:^14} ║\n╚══════════╝",
-        f"╔═══{emoji}═══╗\n║ {message:^12} ║\n╚════════╝",
-        f"╔══{emoji}══╗\n║ {message:^10} ║\n╚══════╝",
-        f"╔═{emoji}═╗\n║ {message:^8} ║\n╚════╝",
-        f"╔{emoji}╗\n║ {message:^6} ║\n╚══╝",
-    ]
-    return frames
-
 # 🕵️‍♂️ Data Fetching and Caching Shenanigans 🕵️‍♂️
 
 def get_rank_papers(url, progress=gr.Progress(track_tqdm=True)):
@@ -65,19 +54,8 @@ def get_rank_papers(url, progress=gr.Progress(track_tqdm=True)):
             
             Github_Star = ppr.find('span', class_='badge badge-secondary').text.strip().replace(',', '') if ppr.find('span', class_='badge badge-secondary') else "0"
             
-            pdf_link = ''
-            try:
-                response_link = session.get(link, headers=headers)
-                soup_link = BeautifulSoup(response_link.text, 'html.parser')
-                paper_info_link = soup_link.find_all('div', class_='paper-abstract')
-                pdf_link = paper_info_link[0].find('div', class_='col-md-12').find('a')['href']
-            except Exception as e:
-                print(f"Failed to retrieve PDF link for {title}: {e}")
-            
-            print(f"Title: {title}, Link: {link}, Github Star: {Github_Star}, PDF Link: {pdf_link}")
-            
             if title not in data_list:
-                data_list[title] = {'link': link, 'Github Star': int(Github_Star), 'pdf_link': pdf_link.strip()}
+                data_list[title] = {'link': link, 'Github Star': int(Github_Star), 'title': title}
             else:
                 break_duplicate -= 1
                 if break_duplicate == 0:
@@ -112,91 +90,53 @@ def load_and_cache_data(url, cache_file):
     save_cached_data(new_data, cache_file)
     return new_data
 
-# 📊 Data Processing and Display Magic 📊
-
-def format_dataframe(data):
-    """Format data into a pretty DataFrame. It's like giving your data a makeover! 💅📈"""
-    if not data:
-        print("No data found to format.")
-        return pd.DataFrame()
-    
-    df = pd.DataFrame(data).T
-    df['title'] = df.index
-    
-    # Check if required columns are present
-    if 'Github Star' in df.columns and 'link' in df.columns and 'pdf_link' in df.columns:
-        df = df[['title', 'Github Star', 'link', 'pdf_link']]
-        df = df.sort_values(by='Github Star', ascending=False)
-        df['link'] = df['link'].apply(lambda x: f'<a href="{x}" target="_blank">Link</a>')
-        df['pdf_link'] = df['pdf_link'].apply(lambda x: f'<a href="{x}" target="_blank">{x}</a>')
-    else:
-        print("Required columns are missing in the dataframe.")
-        print(f"Columns available: {df.columns}")
-    
-    return df
-
-def update_display(category):
-    """Update the display for a category. Freshen up your data like it's spring cleaning! 🧹🌸"""
-    cache_file = f"{category}_papers_cache.json"
-    url = f"https://paperswithcode.com/{category}" if category != "top" else "https://paperswithcode.com/"
-    
-    data = load_and_cache_data(url, cache_file)
-    df = format_dataframe(data)
-    
-    return len(df), df.to_html(escape=False, index=False)
-
-def load_all_data():
-    """Load data for all categories. It's like a buffet for your brain! 🧠🍽️"""
-    top_count, top_html = update_display("top")
-    new_count, new_html = update_display("latest")
-    greatest_count, greatest_html = update_display("greatest")
-    return top_count, top_html, new_count, new_html, greatest_count, greatest_html
-
-# 🚀 Asynchronous Web Page Downloading 🚀
-
-async def download_webpage(session, title, paper_info):
-    """Download the webpage content instead of the PDF. It's like browsing, but faster! 🌐📄"""
-    link_url = paper_info['link']
-    if not link_url:
-        return f"🚫 No link for: {title}. It's playing hide and seek! 🙈", None, None
-    
-    try:
-        timeout = aiohttp.ClientTimeout(total=60)  # 60 seconds timeout
-        async with session.get(link_url, timeout=timeout) as response:
-            if response.status != 200:
-                return f"🚨 Failed to grab webpage for {title}: HTTP {response.status}. The internet gremlins strike again! 👹", None, None
-            page_content = await response.text()
-
-        # Combine the content as a Python type representation
-        code_block = f'"""\nTitle: {title}\nLink: {link_url}\n"""\n\n# Webpage Content\n{repr(page_content)}\n'
-        
-        return f"🎉 Successfully downloaded webpage for: {title}.", code_block, page_content
-    except asyncio.TimeoutError:
-        return f"⏳ Timeout for {title}. The webpage is playing hard to get! 💃", None, None
-    except Exception as e:
-        return f"💥 Oops! Error downloading {title}: {str(e)}. Gremlins in the system! 🛠️", None, None
-
-async def process_webpages(data, progress=gr.Progress()):
-    """Process multiple papers asynchronously by downloading their webpages. 🤹‍♂️🌐"""
-    async with aiohttp.ClientSession() as session:
-        tasks = []
-        for title, paper_info in data.items():
-            task = asyncio.ensure_future(download_webpage(session, title, paper_info))
-            tasks.append(task)
-        
-        results = []
-        codes = []
-        for i, task in enumerate(asyncio.as_completed(tasks), start=1):
-            result, code_block, page_content = await task
-            results.append(result)
-            if code_block:
-                codes.append(code_block)
-            progress(i / len(tasks), f"🚀 Processed {i}/{len(tasks)} papers. Downloading...")
-    
-    return results, codes
-
-def download_all_webpages(progress=gr.Progress()):
-    """Download and display all paper webpages. It's like hosting a web party, and everyone's invited! 🎉🌐"""
+# 🚀 Transformer-based Word and Context Analysis 🚀
+
+def generate_embeddings(titles):
+    """Generate word embeddings using a transformer model."""
+    model_name = "sentence-transformers/all-MiniLM-L6-v2"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModel.from_pretrained(model_name)
+    
+    embeddings = []
+    with torch.no_grad():
+        for title in titles:
+            tokens = tokenizer(title, return_tensors="pt", padding=True, truncation=True)
+            output = model(**tokens)
+            embeddings.append(output.last_hidden_state.mean(dim=1).squeeze())
+    
+    return embeddings
+
+def build_graph(titles, embeddings, threshold=0.7):
+    """Build a graph of words based on similarity between titles."""
+    G = nx.Graph()
+
+    for i, title in enumerate(titles):
+        G.add_node(i, label=title)
+
+    for i in range(len(embeddings)):
+        for j in range(i+1, len(embeddings)):
+            sim = torch.cosine_similarity(embeddings[i], embeddings[j], dim=0).item()
+            if sim > threshold:
+                G.add_edge(i, j, weight=sim)
+
+    return G
+
+def visualize_graph(G, titles):
+    """Visualize the graph using pyvis and show it as a mind map."""
+    net = Network(height="750px", width="100%", notebook=True)
+    
+    for node in G.nodes(data=True):
+        net.add_node(node[0], label=titles[node[0]])
+
+    for edge in G.edges(data=True):
+        net.add_edge(edge[0], edge[1], value=edge[2]['weight'])
+
+    net.show("paper_network.html")
+    return "paper_network.html"
+
+def analyze_and_generate_graph(progress=gr.Progress()):
+    """Analyze papers, generate embeddings, and visualize the relationship graph."""
     all_data = {}
     for category in ["top", "latest", "greatest"]:
         cache_file = f"{category}_papers_cache.json"
@@ -204,19 +144,26 @@ def download_all_webpages(progress=gr.Progress()):
         if data:
             all_data.update(data)
     
-    # Download the webpage content
-    results, code_blocks = asyncio.run(process_webpages(all_data, progress))
+    titles = [paper['title'] for paper in all_data.values()]
     
-    summary = f"📊 Papers processed: {len(all_data)} (We're basically librarians now!)\n"
-    summary += f"✅ Successfully downloaded: {len(code_blocks)} webpages\n"
-    summary += f"❌ Errors: {len(results) - len(code_blocks)} (Even superheroes have off days)\n\n"
+    # Generate embeddings
+    embeddings = generate_embeddings(titles)
     
-    return summary, "\n\n".join(code_blocks)
+    # Build a similarity graph based on the embeddings
+    G = build_graph(titles, embeddings)
+    
+    # Visualize the graph as a mind map
+    graph_file = visualize_graph(G, titles)
+    
+    summary = f"📊 Papers analyzed: {len(titles)}\n"
+    summary += f"✅ Graph generated and visualized.\n"
+    
+    return summary, graph_file
 
 # 🎭 Gradio Interface: Where the Magic Happens 🎭
 
 with gr.Blocks() as demo:
-    gr.Markdown("<h1><center>Papers Leaderboard</center></h1>")
+    gr.Markdown("<h1><center>Papers Leaderboard with Context Analysis</center></h1>")
     
     with gr.Tab("Top Trending Papers"):
         top_count = gr.Textbox(label="Number of Papers Fetched")
@@ -236,14 +183,15 @@ with gr.Blocks() as demo:
         greatest_button = gr.Button("Refresh Leaderboard")
         greatest_button.click(fn=lambda: update_display("greatest"), inputs=None, outputs=[greatest_count, greatest_html])
 
-    download_button = gr.Button("📚 Download All Paper Webpages", variant="primary")
-    download_output = gr.Textbox(label="Download Status")
-    code_output = gr.Code(label="Paper Webpage Contents", language="python")
-    download_button.click(fn=download_all_webpages, inputs=None, outputs=[download_output, code_output])
+    analyze_button = gr.Button("🔍 Analyze and Generate Graph", variant="primary")
+    analyze_output = gr.Textbox(label="Analysis Status")
+    graph_output = gr.HTML(label="Graph Visualization")
+    
+    analyze_button.click(fn=analyze_and_generate_graph, inputs=None, outputs=[analyze_output, graph_output])
 
     # Load initial data for all tabs
     demo.load(fn=load_all_data, outputs=[top_count, top_html, new_count, new_html, greatest_count, greatest_html])
 
 # 🚀 Launch the Gradio interface with a public link
-print("🎭 Launching the Papers Leaderboard! Get ready for a wild ride through the land of academia! 🎢📚")
+print("🎭 Launching the Papers Leaderboard with Context Analysis! Get ready to explore the relationships between papers! 🎢📚")
 demo.launch(share=True)