import requests from bs4 import BeautifulSoup import pandas as pd import gradio as gr import time import os import json def get_rank_papers(url, progress=gr.Progress(track_tqdm=True)): base_url = "https://paperswithcode.com" session = requests.Session() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3', 'Cache-Control': 'no-cache' } print("Time run at : ", time.ctime()) offset = 0 data_list = {} break_duplicate = 10 while True: response = session.get(url, headers=headers, params={'page': offset}) if response.status_code != 200: print('Failed to retrieve data') break soup = BeautifulSoup(response.text, 'html.parser') paper_info = soup.find_all('div', class_='row infinite-item item paper-card') if not paper_info: break for ppr in paper_info: title = ppr.find('h1').text.strip() if "paper" in ppr.find('a')['href']: link = base_url + ppr.find('a')['href'] else: link = ppr.find('a')['href'] Github_Star = ppr.find('span', class_='badge badge-secondary').text.strip().replace(',', '') pdf_link = '' try: response_link = session.get(link, headers=headers) soup_link = BeautifulSoup(response_link.text, 'html.parser') paper_info_link = soup_link.find_all('div', class_='paper-abstract') pdf_link = paper_info_link[0].find('div', class_='col-md-12').find('a')['href'] except: pass if title not in data_list: data_list[title] = {'link': link, 'Github Star': int(Github_Star), 'pdf_link': pdf_link.strip()} else: break_duplicate -= 1 if break_duplicate == 0: return data_list offset += 1 progress.update(offset) print('Data retrieval complete') return data_list def load_cached_data(cache_file): if os.path.exists(cache_file): with open(cache_file, 'r') as f: return json.load(f) return None def save_cached_data(data, cache_file): with open(cache_file, 'w') as f: json.dump(data, f) def format_dataframe(data): df = pd.DataFrame(data).T df['title'] = df.index df = df[['title', 'Github Star', 'link', 'pdf_link']] return df def load_and_cache_data(url, cache_file): cached_data = load_cached_data(cache_file) if cached_data: print(f"Loading cached data from {cache_file}") return cached_data print(f"Fetching new data from {url}") new_data = get_rank_papers(url) save_cached_data(new_data, cache_file) return new_data def update_display(category): cache_file = f"{category}_papers_cache.json" url = f"https://paperswithcode.com/{category}" if category != "top" else "https://paperswithcode.com/" data = load_and_cache_data(url, cache_file) df = format_dataframe(data) return len(df), df def load_all_data(): top_count, top_df = update_display("top") new_count, new_df = update_display("latest") greatest_count, greatest_df = update_display("greatest") return top_count, top_df, new_count, new_df, greatest_count, greatest_df def save_dataframe_generic(df, filename): try: df.to_csv(filename, index=False) return "Dataframe saved successfully." except Exception as e: return f"Error saving dataframe: {e}" def load_dataframe_generic(filename): try: if os.path.exists(filename): df = pd.read_csv(filename) return df, "Dataframe loaded successfully." else: return pd.DataFrame(), "Dataframe file not found." except Exception as e: return pd.DataFrame(), f"Error loading dataframe: {e}" with gr.Blocks() as demo: gr.Markdown("

Papers Leaderboard

") with gr.Tab("Top Trending Papers"): top_count = gr.Textbox(label="Number of Papers Fetched") top_df = gr.DataFrame(interactive=True) top_button = gr.Button("Refresh Leaderboard") top_load_button = gr.Button("Load Dataframe") top_save_button = gr.Button("Save Dataframe") top_save_status = gr.Textbox(label="Status") top_button.click(fn=lambda: update_display("top"), inputs=None, outputs=[top_count, top_df]) top_save_button.click(fn=lambda df: save_dataframe_generic(df, 'top_dataframe.csv'), inputs=top_df, outputs=top_save_status) top_load_button.click(fn=lambda: load_dataframe_generic('top_dataframe.csv'), inputs=None, outputs=[top_df, top_save_status]) with gr.Tab("New Papers"): new_count = gr.Textbox(label="Number of Papers Fetched") new_df = gr.DataFrame(interactive=True) new_button = gr.Button("Refresh Leaderboard") new_load_button = gr.Button("Load Dataframe") new_save_button = gr.Button("Save Dataframe") new_save_status = gr.Textbox(label="Status") new_button.click(fn=lambda: update_display("latest"), inputs=None, outputs=[new_count, new_df]) new_save_button.click(fn=lambda df: save_dataframe_generic(df, 'new_dataframe.csv'), inputs=new_df, outputs=new_save_status) new_load_button.click(fn=lambda: load_dataframe_generic('new_dataframe.csv'), inputs=None, outputs=[new_df, new_save_status]) with gr.Tab("Greatest Papers"): greatest_count = gr.Textbox(label="Number of Papers Fetched") greatest_df = gr.DataFrame(interactive=True) greatest_button = gr.Button("Refresh Leaderboard") greatest_load_button = gr.Button("Load Dataframe") greatest_save_button = gr.Button("Save Dataframe") greatest_save_status = gr.Textbox(label="Status") greatest_button.click(fn=lambda: update_display("greatest"), inputs=None, outputs=[greatest_count, greatest_df]) greatest_save_button.click(fn=lambda df: save_dataframe_generic(df, 'greatest_dataframe.csv'), inputs=greatest_df, outputs=greatest_save_status) greatest_load_button.click(fn=lambda: load_dataframe_generic('greatest_dataframe.csv'), inputs=None, outputs=[greatest_df, greatest_save_status]) # Load initial data for all tabs demo.load(fn=load_all_data, outputs=[top_count, top_df, new_count, new_df, greatest_count, greatest_df]) # Launch the Gradio interface with a public link demo.launch(share=True)