Update app.py

app.py

@@ -11,6 +11,10 @@ import asyncio
 import aiohttp
 import aiofiles
 from concurrent.futures import ThreadPoolExecutor
+import re
+from datetime import datetime
+import zipfile
+import base64
 
 def get_rank_papers(url, progress=gr.Progress(track_tqdm=True)):
     base_url = "https://paperswithcode.com"
@@ -105,51 +109,76 @@ def load_all_data():
     greatest_count, greatest_html = update_display("greatest")
     return top_count, top_html, new_count, new_html, greatest_count, greatest_html
 
-async def download_and_convert_pdf(session, title, paper_info):
+def safe_filename(title):
+    """Convert a string to a safe filename."""
+    return re.sub(r'[^\w\-_\. ]', '_', title)
+
+def create_date_directory():
+    """Create a directory named with the current date."""
+    date_str = datetime.now().strftime("%Y-%m-%d")
+    os.makedirs(date_str, exist_ok=True)
+    return date_str
+
+async def download_and_save_pdf(session, title, paper_info, directory):
     pdf_url = paper_info['pdf_link']
-    cache_file = f"cache/{title.replace(' ', '_')}.md"
-    
-    if os.path.exists(cache_file):
-        async with aiofiles.open(cache_file, 'r') as f:
-            return await f.read()
-    
     if not pdf_url:
-        return f"# {title}\n\nNo PDF link available.\n\n---\n\n"
+        return f"No PDF link available for: {title}"
     
     try:
         async with session.get(pdf_url) as response:
             pdf_content = await response.read()
         
-        pdf_file = io.BytesIO(pdf_content)
-        pdf_reader = PyPDF2.PdfReader(pdf_file)
-        text = ""
-        for page in pdf_reader.pages:
-            text += page.extract_text()
+        safe_title = safe_filename(title)
+        filename = f"{safe_title}.pdf"
+        filepath = os.path.join(directory, filename)
         
-        markdown_text = f"# {title}\n\n{text}\n\n---\n\n"
+        async with aiofiles.open(filepath, 'wb') as f:
+            await f.write(pdf_content)
         
-        os.makedirs('cache', exist_ok=True)
-        async with aiofiles.open(cache_file, 'w') as f:
-            await f.write(markdown_text)
-        
-        return markdown_text
+        return f"Successfully saved: {filename}"
     except Exception as e:
-        return f"# {title}\n\nError processing PDF: {str(e)}\n\n---\n\n"
+        return f"Error saving PDF for {title}: {str(e)}"
 
-async def process_papers(data, progress=gr.Progress()):
+async def process_papers(data, directory, progress=gr.Progress()):
     async with aiohttp.ClientSession() as session:
         tasks = []
         for title, paper_info in data.items():
-            task = asyncio.ensure_future(download_and_convert_pdf(session, title, paper_info))
+            task = asyncio.ensure_future(download_and_save_pdf(session, title, paper_info, directory))
             tasks.append(task)
         
-        consolidated_text = ""
+        results = []
         for i, task in enumerate(asyncio.as_completed(tasks), start=1):
-            markdown_text = await task
-            consolidated_text += markdown_text
+            result = await task
+            results.append(result)
             progress(i / len(tasks), f"Processed {i}/{len(tasks)} papers")
     
-    return consolidated_text
+    return "\n".join(results)
+
+def zip_directory(directory):
+    """Zip the entire directory."""
+    zip_filename = f"{directory}.zip"
+    with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for root, _, files in os.walk(directory):
+            for file in files:
+                zipf.write(os.path.join(root, file), 
+                           os.path.relpath(os.path.join(root, file), 
+                                           os.path.join(directory, '..')))
+    return zip_filename
+
+def get_base64_download_link(file_path):
+    """Create a base64 download link for a file."""
+    with open(file_path, "rb") as file:
+        content = file.read()
+        b64 = base64.b64encode(content).decode()
+        return f'<a href="data:application/zip;base64,{b64}" download="{os.path.basename(file_path)}">Download {os.path.basename(file_path)}</a>'
+
+def get_existing_zip_links():
+    """Get download links for existing zip files."""
+    links = []
+    for file in os.listdir('.'):
+        if file.endswith('.zip') and os.path.isfile(file):
+            links.append(get_base64_download_link(file))
+    return "<br>".join(links)
 
 def download_all_papers(progress=gr.Progress()):
     all_data = {}
@@ -159,12 +188,15 @@ def download_all_papers(progress=gr.Progress()):
         if data:
             all_data.update(data)
     
-    consolidated_text = asyncio.run(process_papers(all_data, progress))
+    date_directory = create_date_directory()
+    results = asyncio.run(process_papers(all_data, date_directory, progress))
+    
+    zip_file = zip_directory(date_directory)
+    download_link = get_base64_download_link(zip_file)
     
-    with open("consolidated_papers.md", "w", encoding="utf-8") as f:
-        f.write(consolidated_text)
+    existing_links = get_existing_zip_links()
     
-    return "All papers have been downloaded and consolidated into 'consolidated_papers.md'", consolidated_text
+    return f"All papers have been downloaded and saved in {zip_file}\n\n{results}", f"{download_link}<br><br>Previous downloads:<br>{existing_links}"
 
 with gr.Blocks() as demo:
     gr.Markdown("<h1><center>Papers Leaderboard</center></h1>")
@@ -189,8 +221,8 @@ with gr.Blocks() as demo:
 
     download_button = gr.Button("📚 Download All Papers", variant="primary")
     download_output = gr.Textbox(label="Download Status")
-    markdown_output = gr.Markdown(label="Paper Content")
-    download_button.click(fn=download_all_papers, inputs=None, outputs=[download_output, markdown_output])
+    download_links = gr.HTML(label="Download Links")
+    download_button.click(fn=download_all_papers, inputs=None, outputs=[download_output, download_links])
 
     # Load initial data for all tabs
     demo.load(fn=load_all_data, outputs=[top_count, top_html, new_count, new_html, greatest_count, greatest_html])