Update app.py

app.py

@@ -122,14 +122,17 @@ def create_date_directory():
 async def download_and_save_pdf(session, title, paper_info, directory):
     pdf_url = paper_info['pdf_link']
     if not pdf_url:
-        return f"No PDF link available for: {title}"
+        return f"No PDF link available for: {title}", None
     
     try:
-        async with session.get(pdf_url) as response:
+        timeout = aiohttp.ClientTimeout(total=60)  # 60 seconds timeout
+        async with session.get(pdf_url, timeout=timeout) as response:
+            if response.status != 200:
+                return f"Failed to download PDF for {title}: HTTP {response.status}", None
             pdf_content = await response.read()
         
         if len(pdf_content) < 2048:  # Check if the PDF is less than 2KB
-            return f"Downloaded PDF for {title} is too small (less than 2KB). Skipping."
+            return f"Downloaded PDF for {title} is too small (less than 2KB). Skipping.", None
         
         safe_title = safe_filename(title)
         filename = f"{safe_title}.pdf"
@@ -138,9 +141,11 @@ async def download_and_save_pdf(session, title, paper_info, directory):
         async with aiofiles.open(filepath, 'wb') as f:
             await f.write(pdf_content)
         
-        return f"Successfully saved: {filename}"
+        return f"Successfully saved: {filename}", filepath
+    except asyncio.TimeoutError:
+        return f"Timeout while downloading PDF for {title}", None
     except Exception as e:
-        return f"Error saving PDF for {title}: {str(e)}"
+        return f"Error saving PDF for {title}: {str(e)}", None
 
 async def process_papers(data, directory, progress=gr.Progress()):
     async with aiohttp.ClientSession() as session:
@@ -150,24 +155,25 @@ async def process_papers(data, directory, progress=gr.Progress()):
             tasks.append(task)
         
         results = []
+        successful_downloads = []
+        errors = []
         for i, task in enumerate(asyncio.as_completed(tasks), start=1):
-            result = await task
+            result, filepath = await task
             results.append(result)
+            if filepath:
+                successful_downloads.append(filepath)
+            else:
+                errors.append(result)
             progress(i / len(tasks), f"Processed {i}/{len(tasks)} papers")
     
-    return "\n".join(results)
+    return results, successful_downloads, errors
 
-def zip_directory(directory):
-    """Zip the entire directory, excluding files smaller than 2KB."""
+def zip_directory(files_to_zip, directory):
+    """Zip the specified files."""
     zip_filename = f"{directory}.zip"
     with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
-        for root, _, files in os.walk(directory):
-            for file in files:
-                file_path = os.path.join(root, file)
-                if os.path.getsize(file_path) >= 2048:  # Only include files 2KB or larger
-                    zipf.write(file_path, 
-                               os.path.relpath(file_path, 
-                                               os.path.join(directory, '..')))
+        for file in files_to_zip:
+            zipf.write(file, os.path.relpath(file, os.path.join(directory, '..')))
     return zip_filename
 
 def get_base64_download_link(file_path):
@@ -194,17 +200,22 @@ def download_all_papers(progress=gr.Progress()):
             all_data.update(data)
     
     date_directory = create_date_directory()
-    results = asyncio.run(process_papers(all_data, date_directory, progress))
+    results, successful_downloads, errors = asyncio.run(process_papers(all_data, date_directory, progress))
     
-    zip_file = zip_directory(date_directory)
-    download_link = get_base64_download_link(zip_file)
+    if successful_downloads:
+        zip_file = zip_directory(successful_downloads, date_directory)
+        download_link = get_base64_download_link(zip_file)
+    else:
+        download_link = "No papers were successfully downloaded."
     
     existing_links = get_existing_zip_links()
     
-    # Count successful downloads
-    successful_downloads = sum(1 for result in results.split('\n') if result.startswith("Successfully saved:"))
+    summary = f"Papers processed: {len(all_data)}\n"
+    summary += f"Successfully downloaded: {len(successful_downloads)}\n"
+    summary += f"Errors: {len(errors)}\n\n"
+    summary += "Error List:\n" + "\n".join(errors) if errors else "No errors occurred."
     
-    return f"Papers downloaded: {successful_downloads} out of {len(all_data)}\nAll papers have been processed and saved in {zip_file}\n\n{results}", f"{download_link}<br><br>Previous downloads:<br>{existing_links}"
+    return summary, f"{download_link}<br><br>Previous downloads:<br>{existing_links}"
 
 with gr.Blocks() as demo:
     gr.Markdown("<h1><center>Papers Leaderboard</center></h1>")