app.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import gradio as gr
import time
import os
import json

def get_rank_papers(url, progress=gr.Progress(track_tqdm=True)):
    base_url = "https://paperswithcode.com"
    session = requests.Session()
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
        'Cache-Control': 'no-cache'
    }
    print("Time run at : ", time.ctime())
    offset = 0
    data_list = {}
    break_duplicate = 10

    while True:
        response = session.get(url, headers=headers, params={'page': offset})
        if response.status_code != 200:
            print('Failed to retrieve data')
            break
        soup = BeautifulSoup(response.text, 'html.parser')
        paper_info = soup.find_all('div', class_='row infinite-item item paper-card')
        if not paper_info:
            break
        for ppr in paper_info:
            title = ppr.find('h1').text.strip()

            if "paper" in ppr.find('a')['href']:
                link = base_url + ppr.find('a')['href']
            else:
                link = ppr.find('a')['href']
            Github_Star = ppr.find('span', class_='badge badge-secondary').text.strip().replace(',', '')
            pdf_link = ''
            try:
                response_link = session.get(link, headers=headers)
                soup_link = BeautifulSoup(response_link.text, 'html.parser')
                paper_info_link = soup_link.find_all('div', class_='paper-abstract')
                pdf_link = paper_info_link[0].find('div', class_='col-md-12').find('a')['href']
            except:
                pass
            if title not in data_list:
                data_list[title] = {'link': link, 'Github Star': int(Github_Star), 'pdf_link': pdf_link.strip()}
            else:
                break_duplicate -= 1
                if break_duplicate == 0:
                    return data_list
        offset += 1
        progress.update(offset)
    print('Data retrieval complete')
    return data_list

def load_cached_data(cache_file):
    if os.path.exists(cache_file):
        with open(cache_file, 'r') as f:
            return json.load(f)
    return None

def save_cached_data(data, cache_file):
    with open(cache_file, 'w') as f:
        json.dump(data, f)

def format_dataframe(data):
    df = pd.DataFrame(data).T
    df['title'] = df.index
    df = df[['title', 'Github Star', 'link', 'pdf_link']]
    return df

def load_and_cache_data(url, cache_file):
    cached_data = load_cached_data(cache_file)

    if cached_data:
        print(f"Loading cached data from {cache_file}")
        return cached_data

    print(f"Fetching new data from {url}")
    new_data = get_rank_papers(url)
    save_cached_data(new_data, cache_file)
    return new_data

def update_display(category):
    cache_file = f"{category}_papers_cache.json"
    url = f"https://paperswithcode.com/{category}" if category != "top" else "https://paperswithcode.com/"

    data = load_and_cache_data(url, cache_file)
    df = format_dataframe(data)

    return len(df), df

def load_all_data():
    top_count, top_df = update_display("top")
    new_count, new_df = update_display("latest")
    greatest_count, greatest_df = update_display("greatest")
    return top_count, top_df, new_count, new_df, greatest_count, greatest_df

def save_dataframe_generic(df, filename):
    try:
        df.to_csv(filename, index=False)
        return "Dataframe saved successfully."
    except Exception as e:
        return f"Error saving dataframe: {e}"

def load_dataframe_generic(filename):
    try:
        if os.path.exists(filename):
            df = pd.read_csv(filename)
            return df, "Dataframe loaded successfully."
        else:
            return pd.DataFrame(), "Dataframe file not found."
    except Exception as e:
        return pd.DataFrame(), f"Error loading dataframe: {e}"

with gr.Blocks() as demo:
    gr.Markdown("<h1><center>Papers Leaderboard</center></h1>")

    with gr.Tab("Top Trending Papers"):
        top_count = gr.Textbox(label="Number of Papers Fetched")
        top_df = gr.DataFrame(interactive=True)
        top_button = gr.Button("Refresh Leaderboard")
        top_load_button = gr.Button("Load Dataframe")
        top_save_button = gr.Button("Save Dataframe")
        top_save_status = gr.Textbox(label="Status")

        top_button.click(fn=lambda: update_display("top"), inputs=None, outputs=[top_count, top_df])
        top_save_button.click(fn=lambda df: save_dataframe_generic(df, 'top_dataframe.csv'), inputs=top_df, outputs=top_save_status)
        top_load_button.click(fn=lambda: load_dataframe_generic('top_dataframe.csv'), inputs=None, outputs=[top_df, top_save_status])

    with gr.Tab("New Papers"):
        new_count = gr.Textbox(label="Number of Papers Fetched")
        new_df = gr.DataFrame(interactive=True)
        new_button = gr.Button("Refresh Leaderboard")
        new_load_button = gr.Button("Load Dataframe")
        new_save_button = gr.Button("Save Dataframe")
        new_save_status = gr.Textbox(label="Status")

        new_button.click(fn=lambda: update_display("latest"), inputs=None, outputs=[new_count, new_df])
        new_save_button.click(fn=lambda df: save_dataframe_generic(df, 'new_dataframe.csv'), inputs=new_df, outputs=new_save_status)
        new_load_button.click(fn=lambda: load_dataframe_generic('new_dataframe.csv'), inputs=None, outputs=[new_df, new_save_status])

    with gr.Tab("Greatest Papers"):
        greatest_count = gr.Textbox(label="Number of Papers Fetched")
        greatest_df = gr.DataFrame(interactive=True)
        greatest_button = gr.Button("Refresh Leaderboard")
        greatest_load_button = gr.Button("Load Dataframe")
        greatest_save_button = gr.Button("Save Dataframe")
        greatest_save_status = gr.Textbox(label="Status")

        greatest_button.click(fn=lambda: update_display("greatest"), inputs=None, outputs=[greatest_count, greatest_df])
        greatest_save_button.click(fn=lambda df: save_dataframe_generic(df, 'greatest_dataframe.csv'), inputs=greatest_df, outputs=greatest_save_status)
        greatest_load_button.click(fn=lambda: load_dataframe_generic('greatest_dataframe.csv'), inputs=None, outputs=[greatest_df, greatest_save_status])

    # Load initial data for all tabs
    demo.load(fn=load_all_data, outputs=[top_count, top_df, new_count, new_df, greatest_count, greatest_df])

# Launch the Gradio interface with a public link
demo.launch(share=True)