Update app.py

app.py

@@ -13,7 +13,98 @@ import aiohttp
 import aiofiles
 from concurrent.futures import ThreadPoolExecutor
 
-# ... (keep the existing functions like get_rank_papers, load_cached_data, save_cached_data, format_dataframe, load_and_cache_data, update_display, load_all_data)
+def get_rank_papers(url, progress=gr.Progress(track_tqdm=True)):
+    base_url = "https://paperswithcode.com"
+    session = requests.Session()
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
+        'Cache-Control': 'no-cache'
+    }
+    print("Time run at : ", time.ctime())
+    offset = 0
+    data_list = {}
+    break_duplicate = 10
+    
+    while True:
+        response = session.get(url, headers=headers, params={'page': offset})
+        if response.status_code != 200:
+            print('Failed to retrieve data')
+            break
+        soup = BeautifulSoup(response.text, 'html.parser')
+        paper_info = soup.find_all('div', class_='row infinite-item item paper-card')
+        if not paper_info:
+            break
+        for ppr in paper_info:
+            title = ppr.find('h1').text.strip()
+            
+            if "paper" in ppr.find('a')['href']:
+                link = base_url + ppr.find('a')['href']
+            else:
+                link = ppr.find('a')['href'] 
+            Github_Star = ppr.find('span', class_='badge badge-secondary').text.strip().replace(',', '')
+            pdf_link = ''
+            try:
+                response_link = session.get(link, headers=headers)
+                soup_link = BeautifulSoup(response_link.text, 'html.parser')
+                paper_info_link = soup_link.find_all('div', class_='paper-abstract')
+                pdf_link = paper_info_link[0].find('div', class_='col-md-12').find('a')['href']
+            except:
+                pass
+            if title not in data_list:
+                data_list[title] = {'link': link, 'Github Star': int(Github_Star), 'pdf_link': pdf_link.strip()}
+            else:
+                break_duplicate -= 1
+                if break_duplicate == 0:
+                    return data_list
+        offset += 1 
+        progress.update(offset)
+    print('Data retrieval complete')   
+    return data_list
+
+def load_cached_data(cache_file):
+    if os.path.exists(cache_file):
+        with open(cache_file, 'r') as f:
+            return json.load(f)
+    return None
+
+def save_cached_data(data, cache_file):
+    with open(cache_file, 'w') as f:
+        json.dump(data, f)
+
+def format_dataframe(data):
+    df = pd.DataFrame(data).T
+    df['title'] = df.index
+    df = df[['title', 'Github Star', 'link', 'pdf_link']]
+    df['link'] = df['link'].apply(lambda x: f'<a href="{x}" target="_blank">Link</a>')
+    df['pdf_link'] = df['pdf_link'].apply(lambda x: f'<a href="{x}" target="_blank">{x}</a>')
+    return df
+
+def load_and_cache_data(url, cache_file):
+    cached_data = load_cached_data(cache_file)
+    
+    if cached_data:
+        print(f"Loading cached data from {cache_file}")
+        return cached_data
+    
+    print(f"Fetching new data from {url}")
+    new_data = get_rank_papers(url)
+    save_cached_data(new_data, cache_file)
+    return new_data
+
+def update_display(category):
+    cache_file = f"{category}_papers_cache.json"
+    url = f"https://paperswithcode.com/{category}" if category != "top" else "https://paperswithcode.com/"
+    
+    data = load_and_cache_data(url, cache_file)
+    df = format_dataframe(data)
+    
+    return len(df), df.to_html(escape=False, index=False)
+
+def load_all_data():
+    top_count, top_html = update_display("top")
+    new_count, new_html = update_display("latest")
+    greatest_count, greatest_html = update_display("greatest")
+    return top_count, top_html, new_count, new_html, greatest_count, greatest_html
 
 async def download_and_convert_pdf(session, title, paper_info):
     pdf_url = paper_info['pdf_link']
@@ -74,7 +165,7 @@ def download_all_papers(progress=gr.Progress()):
     with open("consolidated_papers.md", "w", encoding="utf-8") as f:
         f.write(consolidated_text)
     
-    return "All papers have been downloaded and consolidated into 'consolidated_papers.md'"
+    return "All papers have been downloaded and consolidated into 'consolidated_papers.md'", consolidated_text
 
 with gr.Blocks() as demo:
     gr.Markdown("<h1><center>Papers Leaderboard</center></h1>")