Update app.py

app.py

@@ -5,15 +5,10 @@ import gradio as gr
 import time
 import os
 import json
-import PyPDF2
-import io
-import asyncio
 import aiohttp
 import aiofiles
 import re
 from datetime import datetime
-import base64
-import zipfile
 
 # 🧙‍♂️ Magical Utility Functions 🧙‍♂️
 
@@ -21,19 +16,6 @@ def safe_filename(title):
     """Convert a string to a safe filename. No more 'file not found' nightmares! 🙅‍♂️📁"""
     return re.sub(r'[^\w\-_\. ]', '_', title)
 
-def create_timestamp_directory():
-    """Create a directory named with the current date and time. It's like a time capsule for your downloads! 🗓️📦"""
-    date_str = datetime.now().strftime("%m-%d-%Y-%H-%M")
-    os.makedirs(date_str, exist_ok=True)
-    return date_str
-
-def get_base64_download_link(file_path, filename):
-    """Create a base64 download link for a binary file. It's like teleportation for your files! 🌟📲"""
-    with open(file_path, "rb") as f:
-        content = f.read()
-    b64 = base64.b64encode(content).decode()
-    return f'<a href="data:application/zip;base64,{b64}" download="{filename}">Download {filename}</a>'
-
 # 🎬 Animated Banner Messages 🎬
 def animated_banner(message, emoji):
     """Create an animated banner message. It's like a tiny parade for your console! 🎉🚩"""
@@ -170,73 +152,51 @@ def load_all_data():
     greatest_count, greatest_html = update_display("greatest")
     return top_count, top_html, new_count, new_html, greatest_count, greatest_html
 
-# 🚀 Asynchronous Paper Downloading and Zipping 🚀
+# 🚀 Asynchronous Web Page Downloading 🚀
 
-async def download_pdf(session, title, paper_info, directory):
-    """Download a PDF file. It's like collecting treasures from the internet! 💎📄"""
-    pdf_url = paper_info['pdf_link']
-    if not pdf_url:
-        return f"🚫 No PDF link for: {title}. It's playing hide and seek! 🙈", None
+async def download_webpage(session, title, paper_info):
+    """Download the webpage content instead of the PDF. It's like browsing, but faster! 🌐📄"""
+    link_url = paper_info['link']
+    if not link_url:
+        return f"🚫 No link for: {title}. It's playing hide and seek! 🙈", None, None
     
     try:
         timeout = aiohttp.ClientTimeout(total=60)  # 60 seconds timeout
-        async with session.get(pdf_url, timeout=timeout) as response:
+        async with session.get(link_url, timeout=timeout) as response:
             if response.status != 200:
-                return f"🚨 Failed to grab PDF for {title}: HTTP {response.status}. The internet gremlins strike again! 👹", None
-            pdf_content = await response.read()
-
-        file_length = len(pdf_content)
-        if file_length < 5000:  # Check if the PDF is less than 5KB
-            return f"🐜 PDF for {title} is tiny ({file_length} bytes). It's like a paper for ants! 🐜📄", None
+                return f"🚨 Failed to grab webpage for {title}: HTTP {response.status}. The internet gremlins strike again! 👹", None, None
+            page_content = await response.text()
 
-        # Save the PDF to the directory
-        safe_title = safe_filename(title)
-        pdf_filename = f"{safe_title}.pdf"
-        pdf_filepath = os.path.join(directory, pdf_filename)
+        # Combine the content as a Python type representation
+        code_block = f'"""\nTitle: {title}\nLink: {link_url}\n"""\n\n# Webpage Content\n{repr(page_content)}\n'
         
-        async with aiofiles.open(pdf_filepath, 'wb') as f:
-            await f.write(pdf_content)
-
-        return f"🎉 Successfully downloaded: {pdf_filename} (File length: {file_length} bytes).", pdf_filepath
+        return f"🎉 Successfully downloaded webpage for: {title}.", code_block, page_content
     except asyncio.TimeoutError:
-        return f"⏳ Timeout for {title}. The PDF is playing hard to get! 💃", None
+        return f"⏳ Timeout for {title}. The webpage is playing hard to get! 💃", None, None
     except Exception as e:
-        return f"💥 Oops! Error downloading {title}: {str(e)}. Gremlins in the system! 🛠️", None
+        return f"💥 Oops! Error downloading {title}: {str(e)}. Gremlins in the system! 🛠️", None, None
 
-async def process_papers(data, directory, progress=gr.Progress()):
-    """Process multiple papers asynchronously by downloading their PDFs. 🤹‍♂️📚"""
+async def process_webpages(data, progress=gr.Progress()):
+    """Process multiple papers asynchronously by downloading their webpages. 🤹‍♂️🌐"""
     async with aiohttp.ClientSession() as session:
         tasks = []
         for title, paper_info in data.items():
-            task = asyncio.ensure_future(download_pdf(session, title, paper_info, directory))
+            task = asyncio.ensure_future(download_webpage(session, title, paper_info))
             tasks.append(task)
         
         results = []
-        pdf_files = []
+        codes = []
         for i, task in enumerate(asyncio.as_completed(tasks), start=1):
-            result, pdf_filepath = await task
+            result, code_block, page_content = await task
             results.append(result)
-            if pdf_filepath:
-                pdf_files.append(pdf_filepath)
+            if code_block:
+                codes.append(code_block)
             progress(i / len(tasks), f"🚀 Processed {i}/{len(tasks)} papers. Downloading...")
     
-    return results, pdf_files
-
-def zip_files(directory):
-    """Zip all files in the given directory."""
-    zip_filename = f"{directory}.zip"
-    zip_filepath = os.path.join(directory, zip_filename)
-    
-    with zipfile.ZipFile(zip_filepath, 'w') as zipf:
-        for root, _, files in os.walk(directory):
-            for file in files:
-                file_path = os.path.join(root, file)
-                zipf.write(file_path, arcname=file)
-    
-    return zip_filepath
+    return results, codes
 
-def download_all_papers(progress=gr.Progress()):
-    """Download and zip all papers. It's like hosting a paper party, and everyone's invited! 🎉📚"""
+def download_all_webpages(progress=gr.Progress()):
+    """Download and display all paper webpages. It's like hosting a web party, and everyone's invited! 🎉🌐"""
     all_data = {}
     for category in ["top", "latest", "greatest"]:
         cache_file = f"{category}_papers_cache.json"
@@ -244,24 +204,14 @@ def download_all_papers(progress=gr.Progress()):
         if data:
             all_data.update(data)
     
-    # Create timestamped directory
-    date_directory = create_timestamp_directory()
-    
-    # Download papers
-    results, pdf_files = asyncio.run(process_papers(all_data, date_directory, progress))
-    
-    # Zip the directory
-    zip_filepath = zip_files(date_directory)
-    
-    # Create a download link
-    download_link = get_base64_download_link(zip_filepath, f"{date_directory}.zip")
+    # Download the webpage content
+    results, code_blocks = asyncio.run(process_webpages(all_data, progress))
     
     summary = f"📊 Papers processed: {len(all_data)} (We're basically librarians now!)\n"
-    summary += f"✅ Successfully downloaded and zipped: {len(pdf_files)} (Take that, PDF gremlins!)\n"
-    summary += f"❌ Errors: {len(results) - len(pdf_files)} (Even superheroes have off days)\n\n"
-    summary += download_link
+    summary += f"✅ Successfully downloaded: {len(code_blocks)} webpages\n"
+    summary += f"❌ Errors: {len(results) - len(code_blocks)} (Even superheroes have off days)\n\n"
     
-    return summary, "<br>".join(results)
+    return summary, "\n\n".join(code_blocks)
 
 # 🎭 Gradio Interface: Where the Magic Happens 🎭
 
@@ -286,11 +236,10 @@ with gr.Blocks() as demo:
         greatest_button = gr.Button("Refresh Leaderboard")
         greatest_button.click(fn=lambda: update_display("greatest"), inputs=None, outputs=[greatest_count, greatest_html])
 
-    download_button = gr.Button("📚 Download All Papers", variant="primary")
+    download_button = gr.Button("📚 Download All Paper Webpages", variant="primary")
     download_output = gr.Textbox(label="Download Status")
-    download_links = gr.HTML(label="Download Links")
-    text_output = gr.Code(label="Paper Contents", language="python")
-    download_button.click(fn=download_all_papers, inputs=None, outputs=[download_output, download_links, text_output])
+    code_output = gr.Code(label="Paper Webpage Contents", language="python")
+    download_button.click(fn=download_all_webpages, inputs=None, outputs=[download_output, code_output])
 
     # Load initial data for all tabs
     demo.load(fn=load_all_data, outputs=[top_count, top_html, new_count, new_html, greatest_count, greatest_html])