Terry Zhuo
Merge branch 'main' of https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard
1e748fb
| # some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main | |
| import json | |
| import os | |
| from datetime import datetime, timezone | |
| import gradio as gr | |
| import pandas as pd | |
| import requests | |
| from huggingface_hub import HfApi | |
| from src.css_html import custom_css | |
| from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT_3 | |
| from src.utils import ( | |
| AutoEvalColumn, | |
| fields, | |
| is_model_on_hub, | |
| make_clickable_names, | |
| plot_elo_mle, | |
| plot_solve_rate, | |
| styled_error, | |
| styled_message, | |
| ) | |
| from datasets import load_dataset | |
| TOKEN = os.environ.get("TOKEN", None) | |
| api = HfApi(TOKEN) | |
| df = load_dataset("bigcode/bigcodebench-results", split="train").to_pandas().sort_values("complete", ascending=False) | |
| task_elo_mle_df = load_dataset("bigcode/bigcodebench-elo", split="train").to_pandas() | |
| model_elo_mle_df = load_dataset("bigcode/bigcodebench-elo-model-with-tie", split="train").to_pandas() | |
| complete_solve_rate = load_dataset("bigcode/bigcodebench-complete-solve-rate", split="train").to_pandas() | |
| instruct_solve_rate = load_dataset("bigcode/bigcodebench-instruct-solve-rate", split="train").to_pandas() | |
| QUEUE_REPO = "bigcode/bigcodebench-requests" | |
| EVAL_REQUESTS_PATH = "eval-queue" | |
| COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] | |
| TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden] | |
| COLS_LITE = [ | |
| c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden | |
| ] | |
| TYPES_LITE = [ | |
| c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden | |
| ] | |
| def add_new_eval( | |
| model: str, | |
| revision: str, | |
| model_type: str, | |
| ): | |
| current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | |
| if model_type is None or model_type == "": | |
| return styled_error("Please select a model type.") | |
| # check the model actually exists before adding the eval | |
| if revision == "": | |
| revision = "main" | |
| model_on_hub, error = is_model_on_hub(model, revision) | |
| if not model_on_hub: | |
| return styled_error(f'Model "{model}" {error}') | |
| print("adding new eval") | |
| eval_entry = { | |
| "model": model, | |
| "revision": revision, | |
| "status": "PENDING", | |
| "submitted_time": current_time, | |
| "model_type": model_type.split(" ")[1], | |
| } | |
| user_name = "" | |
| model_path = model | |
| if "/" in model: | |
| user_name = model.split("/")[0] | |
| model_path = model.split("/")[1] | |
| OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}" | |
| os.makedirs(OUT_DIR, exist_ok=True) | |
| out_path = f"{OUT_DIR}/{model_path}_eval_request.json" | |
| print(f"Saving eval request to {out_path}") | |
| with open(out_path, "w") as f: | |
| f.write(json.dumps(eval_entry)) | |
| api.upload_file( | |
| path_or_fileobj=out_path, | |
| path_in_repo=out_path.split("eval-queue/")[1], | |
| repo_id=QUEUE_REPO, | |
| repo_type="dataset", | |
| commit_message=f"Add {model} to eval queue", | |
| ) | |
| # remove the local file | |
| os.remove(out_path) | |
| return styled_message("Your request has been submitted to the evaluation queue!\n") | |
| def select_columns(df, columns): | |
| always_here_cols = [ | |
| AutoEvalColumn.model_type_symbol.name, | |
| AutoEvalColumn.model.name, | |
| ] | |
| # We use COLS to maintain sorting | |
| filtered_df = df[ | |
| always_here_cols + [c for c in COLS if c in df.columns and c in columns] | |
| ] | |
| return filtered_df | |
| def filter_items(df, leaderboard_table, query): | |
| if query == "all": | |
| return df[leaderboard_table.columns] | |
| else: | |
| query = query[0] | |
| filtered_df = df[df["type"].str.contains(query, na=False)] | |
| return filtered_df[leaderboard_table.columns] | |
| def search_table(df, leaderboard_table, query): | |
| filtered_df = df[(df["model"].str.contains(query, case=False))] | |
| return filtered_df[leaderboard_table.columns] | |
| df = make_clickable_names(df) | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| with gr.Row(): | |
| gr.Markdown( | |
| """<div style="text-align: center;"><h1> 🌸<span style='color: #A74E95;'>Big</span><span style='color: #C867B5;'>Code</span><span style='color: #DD71C8;'>Bench</span> Leaderboard🌸</h1></div>\ | |
| <br>\ | |
| <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">⭐ Big Code Models Leaderboard</a>, we compare performance of LLMs on <a href="https://huggingface.co/datasets/bigcode/bigcodebench">BigCodeBench</a> benchmark.</p> | |
| """, | |
| elem_classes="markdown-text", | |
| ) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.Column(): | |
| with gr.Tabs(elem_classes="A100-tabs") as A100_tabs: | |
| with gr.TabItem("🔍 Evaluation Table", id=0): | |
| with gr.Column(): | |
| with gr.Accordion("➡️ See All Columns", open=False): | |
| shown_columns = gr.CheckboxGroup( | |
| choices=[ | |
| c | |
| for c in COLS | |
| if c | |
| not in [ | |
| AutoEvalColumn.dummy.name, | |
| AutoEvalColumn.model.name, | |
| AutoEvalColumn.model_type_symbol.name, | |
| ] | |
| ], | |
| value=[ | |
| c | |
| for c in COLS_LITE | |
| if c | |
| not in [ | |
| AutoEvalColumn.dummy.name, | |
| AutoEvalColumn.model.name, | |
| AutoEvalColumn.model_type_symbol.name, | |
| ] | |
| ], | |
| label="", | |
| elem_id="column-select", | |
| interactive=True, | |
| ) | |
| # with gr.Column(min_width=780): | |
| with gr.Row(): | |
| search_bar = gr.Textbox( | |
| placeholder="🔍 Search for your model and press ENTER...", | |
| show_label=False, | |
| elem_id="search-bar", | |
| ) | |
| filter_columns = gr.Radio( | |
| label="⏚ Filter model types", | |
| choices=["all", "🟢 base", "🔶 instruction-tuned", "EXT external-evaluation"], | |
| value="all", | |
| elem_id="filter-columns", | |
| ) | |
| leaderboard_df = gr.components.Dataframe( | |
| value=df[ | |
| [ | |
| AutoEvalColumn.model_type_symbol.name, | |
| AutoEvalColumn.model.name, | |
| ] | |
| + shown_columns.value | |
| ], | |
| headers=[ | |
| AutoEvalColumn.model_type_symbol.name, | |
| AutoEvalColumn.model.name, | |
| ] | |
| + shown_columns.value, | |
| datatype=TYPES, | |
| elem_id="leaderboard-table", | |
| interactive=False, | |
| ) | |
| hidden_leaderboard_df = gr.components.Dataframe( | |
| value=df, | |
| headers=COLS, | |
| datatype=["str" for _ in range(len(COLS))], | |
| visible=False, | |
| ) | |
| search_bar.submit( | |
| search_table, | |
| [hidden_leaderboard_df, leaderboard_df, search_bar], | |
| leaderboard_df, | |
| ) | |
| filter_columns.change( | |
| filter_items, | |
| [hidden_leaderboard_df, leaderboard_df, filter_columns], | |
| leaderboard_df, | |
| ) | |
| shown_columns.change( | |
| select_columns, | |
| [hidden_leaderboard_df, shown_columns], | |
| leaderboard_df, | |
| ) | |
| gr.Markdown( | |
| """ | |
| **Notes:** | |
| - _Complete_ vs _Instruct_: | |
| - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This variant tests if the models are good at coding. | |
| - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This variant tests if the models are really capable enough to understand human intents to code. | |
| - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants. | |
| - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on `BigCodeBench-Complete`, which starts from 1000 and is boostrapped 500 times. | |
| - `size` is the amount of activated model weight during inference. | |
| - Some instruction-tuned models are marked with 🟢 symbol, as they miss the chat templates in their tokenizer configurations. | |
| - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination. | |
| - For more details check the 📝 About section. | |
| - Models with a 🔴 symbol represent external evaluation submission, this means that we didn't verify the results, you can find the author's submission under `Submission PR` field from `See All Columns` tab. | |
| """, | |
| elem_classes="markdown-text", | |
| ) | |
| with gr.TabItem("📊 Elo Rating", id=1): | |
| with gr.Column(): | |
| with gr.Group(): | |
| gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_") | |
| task_elo_map = gr.Plot() | |
| demo.load(plot_elo_mle, [gr.Dataframe(task_elo_mle_df, visible=False)], task_elo_map) | |
| with gr.Group(): | |
| gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)") | |
| model_elo_map = gr.Plot() | |
| demo.load(plot_elo_mle, [gr.Dataframe(model_elo_mle_df, visible=False)], model_elo_map) | |
| with gr.TabItem("🧩 Solve Rate", id=2): | |
| with gr.Column(): | |
| complete_map = gr.Plot() | |
| demo.load(plot_solve_rate, [gr.Dataframe(complete_solve_rate, visible=False), | |
| gr.Textbox("Complete", visible=False), | |
| ], complete_map) | |
| instruct_map = gr.Plot() | |
| demo.load(plot_solve_rate, [gr.Dataframe(instruct_solve_rate, visible=False), | |
| gr.Textbox("Instruct", visible=False), | |
| ], instruct_map) | |
| with gr.TabItem("📝 About", id=3): | |
| gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text") | |
| with gr.TabItem("Submit Results 🚀", id=4): | |
| gr.Markdown(SUBMISSION_TEXT_3) | |
| demo.launch() | |