|
|
import gradio as gr |
|
|
import pandas as pd |
|
|
import numpy as np |
|
|
|
|
|
|
|
|
data = { |
|
|
"model": [ |
|
|
"GPT-4o", "GPT-4o-mini", "Gemini-2.0-Flash", "Qwen2-VL", "Qwen2.5-VL", |
|
|
"AIN-7B", "Tesseract", "EasyOCR", "Paddle", "Surya", |
|
|
"AzureOCR", "Qaari", "Gemma3", "ArabicNougat" |
|
|
], |
|
|
"organization": [ |
|
|
"OpenAI", "OpenAI", "Google", "Alibaba", "Alibaba", |
|
|
"MBZUAI", "Google", "JaidedAI", "Baidu", "VikParuchuri", |
|
|
"Microsoft", "NAMAA", "Google", "MohamedRashad" |
|
|
], |
|
|
"type": [ |
|
|
"Closed-source", "Closed-source", "Closed-source", "Open-source", "Open-source", |
|
|
"Open-source", "Open-source", "Open-source", "Open-source", "Open-source", |
|
|
"Closed-source", "Open-source", "Open-source", "Open-source" |
|
|
], |
|
|
"task": [ |
|
|
"OCR/Vision", "OCR/Vision", "OCR/Vision", "OCR/Vision", "OCR/Vision", |
|
|
"OCR/Vision", "OCR", "OCR", "OCR", "OCR/Arabic", |
|
|
"OCR/Vision", "OCR/Arabic", "OCR/Vision", "OCR/Document" |
|
|
], |
|
|
"chrf": [ |
|
|
61.01, 47.21, 77.95, 33.94, 49.23, |
|
|
78.33, 39.62, 45.47, 16.73, 20.61, |
|
|
50.97, 39.77, 30.02, 30.52 |
|
|
], |
|
|
"cer": [ |
|
|
0.31, 0.43, 0.13, 1.48, 1.20, |
|
|
0.20, 0.54, 0.58, 0.79, 4.95, |
|
|
0.52, 1.80, 1.05, 4.37 |
|
|
], |
|
|
"wer": [ |
|
|
0.55, 0.71, 0.32, 1.55, 1.41, |
|
|
0.28, 0.84, 0.89, 1.02, 5.61, |
|
|
0.69, 1.93, 1.45, 4.67 |
|
|
], |
|
|
"downloads": [ |
|
|
"-", "-", "-", "1262K", "3313K", |
|
|
"0.86K", "-", "-", "-", "39K", |
|
|
"-", "5K", "227K", "0.75K" |
|
|
], |
|
|
"model_url": [ |
|
|
"https://openai.com/index/hello-gpt-4o/", |
|
|
"https://platform.openai.com/docs/models/gpt-4o-mini", |
|
|
"https://deepmind.google/technologies/gemini/flash/", |
|
|
"https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct", |
|
|
"https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct", |
|
|
"https://huggingface.co/MBZUAI/AIN", |
|
|
"https://github.com/tesseract-ocr/tesseract", |
|
|
"https://github.com/JaidedAI/EasyOCR", |
|
|
"https://github.com/PaddlePaddle/PaddleOCR", |
|
|
"https://github.com/VikParuchuri/surya", |
|
|
"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-ocr", |
|
|
"https://huggingface.co/NAMAA-Space/Qari-OCR-0.1-VL-2B-Instruct", |
|
|
"https://huggingface.co/google/gemma-3-12b-it", |
|
|
"https://huggingface.co/MohamedRashad/arabic-large-nougat" |
|
|
], |
|
|
"paper_url": [ |
|
|
"https://arxiv.org/abs/2410.21276", |
|
|
"https://arxiv.org/abs/2410.21276", |
|
|
"https://blog.google/technology/google-deepmind/google-gemini-ai-update-december-2024/", |
|
|
"https://arxiv.org/abs/2409.12191", |
|
|
"https://arxiv.org/abs/2502.13923", |
|
|
"https://arxiv.org/abs/2502.00094", |
|
|
"https://github.com/tesseract-ocr/tesseract", |
|
|
"https://github.com/JaidedAI/EasyOCR", |
|
|
"https://arxiv.org/abs/2206.03001", |
|
|
"https://github.com/VikParuchuri/surya", |
|
|
"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/overview-ocr", |
|
|
"https://huggingface.co/NAMAA-Space/Qari-OCR-0.1-VL-2B-Instruct", |
|
|
"https://developers.googleblog.com/en/introducing-gemma3/", |
|
|
"https://arxiv.org/abs/2411.17835" |
|
|
] |
|
|
} |
|
|
|
|
|
|
|
|
df = pd.DataFrame(data) |
|
|
|
|
|
def format_dataframe(df): |
|
|
formatted_df = df.copy() |
|
|
formatted_df['chrf'] = formatted_df['chrf'].apply( |
|
|
lambda x: f"<span style='color: {'#10B981' if x > 60 else '#F59E0B' if x > 40 else '#EF4444'}'>{x:.1f}</span>" |
|
|
) |
|
|
formatted_df['cer'] = formatted_df['cer'].apply( |
|
|
lambda x: f"<span style='color: {'#10B981' if x < 0.5 else '#F59E0B' if x < 1 else '#EF4444'}'>{x:.2f}</span>" |
|
|
) |
|
|
formatted_df['wer'] = formatted_df['wer'].apply( |
|
|
lambda x: f"<span style='color: {'#10B981' if x < 0.5 else '#F59E0B' if x < 1 else '#EF4444'}'>{x:.2f}</span>" |
|
|
) |
|
|
formatted_df['model'] = formatted_df.apply( |
|
|
lambda row: f"<a href='{row['model_url']}' target='_blank'>{row['model']}</a>", axis=1 |
|
|
) |
|
|
formatted_df['paper'] = formatted_df.apply( |
|
|
lambda row: f"<a href='{row['paper_url']}' target='_blank'>Paper</a>", axis=1 |
|
|
) |
|
|
formatted_df['type'] = formatted_df['type'].apply( |
|
|
lambda x: f"<span style='background-color: {'#DBEAFE' if x == 'Open-source' else '#FEF3C7'}; padding: 2px 6px; border-radius: 9999px; font-size: 0.75rem;'>{x}</span>" |
|
|
) |
|
|
formatted_df['task'] = formatted_df['task'].apply( |
|
|
lambda x: f"<span style='background-color: #E0F2FE; padding: 2px 6px; border-radius: 9999px; font-size: 0.75rem;'>{x}</span>" |
|
|
) |
|
|
formatted_df = formatted_df.drop(columns=['model_url', 'paper_url']) |
|
|
return formatted_df |
|
|
|
|
|
css = """ |
|
|
#leaderboard-title { |
|
|
text-align: center; |
|
|
margin-bottom: 0; |
|
|
} |
|
|
#leaderboard-subtitle { |
|
|
text-align: center; |
|
|
margin-top: 0; |
|
|
color: #6B7280; |
|
|
font-size: 1rem; |
|
|
} |
|
|
.gradio-container { |
|
|
max-width: 1200px !important; |
|
|
} |
|
|
.hf-logo { |
|
|
display: flex; |
|
|
align-items: center; |
|
|
justify-content: center; |
|
|
margin-bottom: 1rem; |
|
|
} |
|
|
.hf-logo img { |
|
|
height: 50px; |
|
|
} |
|
|
.header { |
|
|
background: linear-gradient(90deg, #FFDE59 0%, #FFC532 100%); |
|
|
padding: 20px; |
|
|
border-radius: 8px; |
|
|
margin-bottom: 20px; |
|
|
display: flex; |
|
|
align-items: center; |
|
|
justify-content: space-between; |
|
|
} |
|
|
.header img { |
|
|
height: 40px; |
|
|
margin-right: 15px; |
|
|
} |
|
|
.header-content { |
|
|
display: flex; |
|
|
align-items: center; |
|
|
} |
|
|
.header-text { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
} |
|
|
.header-text h1 { |
|
|
margin: 0; |
|
|
font-size: 1.5rem; |
|
|
font-weight: bold; |
|
|
color: black; |
|
|
} |
|
|
.header-text p { |
|
|
margin: 0; |
|
|
color: rgba(0, 0, 0, 0.8); |
|
|
} |
|
|
.filter-container { |
|
|
display: flex; |
|
|
flex-wrap: wrap; |
|
|
gap: 10px; |
|
|
margin-bottom: 20px; |
|
|
} |
|
|
table { |
|
|
width: 100%; |
|
|
border-collapse: collapse; |
|
|
} |
|
|
th { |
|
|
background-color: #F9FAFB; |
|
|
text-align: left; |
|
|
padding: 12px; |
|
|
font-weight: 600; |
|
|
color: #374151; |
|
|
border-bottom: 1px solid #E5E7EB; |
|
|
} |
|
|
td { |
|
|
padding: 12px; |
|
|
border-bottom: 1px solid #E5E7EB; |
|
|
} |
|
|
tr:hover { |
|
|
background-color: #F9FAFB; |
|
|
} |
|
|
a { |
|
|
color: #2563EB; |
|
|
text-decoration: none; |
|
|
} |
|
|
a:hover { |
|
|
text-decoration: underline; |
|
|
} |
|
|
.footer { |
|
|
display: flex; |
|
|
justify-content: space-between; |
|
|
align-items: center; |
|
|
padding: 10px 0; |
|
|
color: #6B7280; |
|
|
font-size: 0.875rem; |
|
|
margin-top: 20px; |
|
|
} |
|
|
.footer a { |
|
|
color: #2563EB; |
|
|
text-decoration: none; |
|
|
display: inline-flex; |
|
|
align-items: center; |
|
|
} |
|
|
.footer a:hover { |
|
|
text-decoration: underline; |
|
|
} |
|
|
""" |
|
|
|
|
|
|
|
|
hf_logo = """ |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="120" height="40" viewBox="0 0 95 25" fill="none"> |
|
|
<path d="M8.51825 0H11.3583V17.7547H8.51825V0Z" fill="black"/> |
|
|
<path d="M30.1975 5.07422H33.0375V17.7547H30.1975V16.2969C28.9408 17.4158 27.6842 18.0602 25.94 18.0602C22.455 18.0602 19.5825 15.1877 19.5825 11.4358C19.5825 7.6839 22.455 4.8114 25.94 4.8114C27.6842 4.8114 28.9408 5.4558 30.1975 6.5747V5.07422ZM26.2882 15.403C28.7225 15.403 30.1975 13.7014 30.1975 11.4358C30.1975 9.1702 28.7225 7.4686 26.2882 7.4686C23.8539 7.4686 22.3789 9.1702 22.3789 11.4358C22.3789 13.7014 23.8539 15.403 26.2882 15.403Z" fill="black"/> |
|
|
<path d="M35.1311 11.4358C35.1311 7.6839 38.0036 4.8114 41.7555 4.8114C45.5075 4.8114 48.38 7.6839 48.38 11.4358C48.38 15.1877 45.5075 18.0602 41.7555 18.0602C38.0036 18.0602 35.1311 15.1877 35.1311 11.4358ZM45.5839 11.4358C45.5839 9.1702 44.1089 7.4686 41.7555 7.4686C39.402 7.4686 37.927 9.1702 37.927 11.4358C37.927 13.7014 39.402 15.403 41.7555 15.403C44.1089 15.403 45.5839 13.7014 45.5839 11.4358Z" fill="black"/> |
|
|
<path d="M50.2717 0H53.1117V17.7547H50.2717V0Z" fill="black"/> |
|
|
<path d="M55.1956 0H58.0356V17.7547H55.1956V0Z" fill="black"/> |
|
|
<path d="M68.3864 11.4359C68.3864 9.0824 66.9114 7.4686 64.558 7.4686C62.2046 7.4686 60.6521 9.0824 60.6521 11.4359C60.6521 13.7893 62.2047 15.4031 64.558 15.4031C66.9114 15.4031 68.3864 13.7893 68.3864 11.4359ZM57.8122 11.4359C57.8122 7.6839 60.6847 4.8114 64.4367 4.8114C66.1809 4.8114 67.4374 5.45579 68.6939 6.57469V5.07422H71.5341V18.0602C71.5341 22.1174 68.5725 24.618 64.5575 24.618C61.2553 24.618 58.5041 22.8739 57.7383 20.0013L60.5347 19.1142C61.0577 20.6146 62.5748 21.9605 64.5575 21.9605C66.9111 21.9605 68.6941 20.5285 68.6941 18.0602V16.297C67.4374 17.4159 66.1809 18.0603 64.4367 18.0603C60.6847 18.0603 57.8122 15.1878 57.8122 11.4359Z" fill="black"/> |
|
|
<path d="M74.0307 11.4358C74.0307 7.6839 76.9032 4.8114 80.6551 4.8114C84.4071 4.8114 87.2796 7.6839 87.2796 11.4358C87.2796 15.1877 84.4071 18.0602 80.6551 18.0602C76.9032 18.0602 74.0307 15.1877 74.0307 11.4358ZM84.4835 11.4358C84.4835 9.1702 83.0085 7.4686 80.6551 7.4686C78.3016 7.4686 76.8266 9.1702 76.8266 11.4358C76.8266 13.7014 78.3016 15.403 80.6551 15.403C83.0085 15.403 84.4835 13.7014 84.4835 11.4358Z" fill="black"/> |
|
|
<path d="M89.9903 2.69156C89.9903 1.63531 90.7989 0.82666 91.8551 0.82666C92.9114 0.82666 93.72 1.63531 93.72 2.69156C93.72 3.74781 92.9114 4.55645 91.8551 4.55645C90.7989 4.55645 89.9903 3.74781 89.9903 2.69156ZM90.1952 5.07422H93.5149V17.7547H90.1952V5.07422Z" fill="black"/> |
|
|
</svg> |
|
|
""" |
|
|
|
|
|
def filter_by_type(df, type_filter): |
|
|
if type_filter == "All": |
|
|
return df |
|
|
return df[df["type"].str.contains(type_filter)] |
|
|
|
|
|
def filter_by_search(df, search_term): |
|
|
if not search_term: |
|
|
return df |
|
|
search_term = search_term.lower() |
|
|
mask = ( |
|
|
df["model"].str.lower().str.contains(search_term) | |
|
|
df["organization"].str.lower().str.contains(search_term) | |
|
|
df["task"].str.lower().str.contains(search_term) |
|
|
) |
|
|
|
|
|
return df[mask] |
|
|
|
|
|
def create_leaderboard_interface(): |
|
|
df_orig = pd.DataFrame(data) |
|
|
df_orig = df_orig.sort_values(by="cer", ascending=True) |
|
|
|
|
|
with gr.Blocks(css=css) as demo: |
|
|
gr.HTML(f""" |
|
|
<div class="header"> |
|
|
<div class="header-content"> |
|
|
<div> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" viewBox="0 0 40 40" fill="none"> |
|
|
<path d="M9 16H11V24H9V16Z" fill="black"/> |
|
|
<path d="M13 11H15V29H13V11Z" fill="black"/> |
|
|
<path d="M17 15H19V25H17V15Z" fill="black"/> |
|
|
<path d="M21 11H23V29H21V11Z" fill="black"/> |
|
|
<path d="M25 16H27V24H25V16Z" fill="black"/> |
|
|
<path d="M29 14H31V26H29V14Z" fill="black"/> |
|
|
</svg> |
|
|
</div> |
|
|
<div class="header-text"> |
|
|
<h1>KITAB-Bench Leaderboard</h1> |
|
|
<p>Arabic OCR and Document Understanding Benchmark</p> |
|
|
</div> |
|
|
</div> |
|
|
<div> |
|
|
<a href="https://huggingface.co/spaces" target="_blank" style="color: black; text-decoration: underline;"> |
|
|
Powered by 🤗 Spaces |
|
|
</a> |
|
|
</div> |
|
|
</div> |
|
|
""") |
|
|
|
|
|
|
|
|
with gr.Row(equal_height=True): |
|
|
type_filter = gr.Radio( |
|
|
["All", "Open-source", "Closed-source"], |
|
|
label="Model Type", |
|
|
value="All", |
|
|
interactive=True |
|
|
) |
|
|
search_input = gr.Textbox( |
|
|
label="Search Models, Organizations, or Tasks", |
|
|
placeholder="Type to search...", |
|
|
interactive=True |
|
|
) |
|
|
|
|
|
sort_by = gr.Dropdown( |
|
|
["chrf", "cer", "wer"], |
|
|
label="Sort by", |
|
|
value="chrf", |
|
|
interactive=True |
|
|
) |
|
|
|
|
|
sort_order = gr.Radio( |
|
|
["Descending", "Ascending"], |
|
|
label="Sort Order", |
|
|
value="Ascending", |
|
|
interactive=True |
|
|
) |
|
|
|
|
|
table_output = gr.HTML() |
|
|
def update_table(type_filter, search_term, sort_by, sort_order): |
|
|
filtered_df = filter_by_type(df_orig, type_filter) |
|
|
filtered_df = filter_by_search(filtered_df, search_term) |
|
|
is_ascending = sort_order == "Ascending" |
|
|
if sort_by == "chrf": |
|
|
is_ascending = not is_ascending |
|
|
|
|
|
filtered_df = filtered_df.sort_values(by=sort_by, ascending=is_ascending) |
|
|
formatted_df = format_dataframe(filtered_df) |
|
|
|
|
|
html_table = f""" |
|
|
<div style="overflow-x: auto;"> |
|
|
<table style="width:100%"> |
|
|
<thead> |
|
|
<tr> |
|
|
<th>Model</th> |
|
|
<th>Organization</th> |
|
|
<th>Type</th> |
|
|
<th>Task</th> |
|
|
<th>CHrF ↑</th> |
|
|
<th>CER ↓</th> |
|
|
<th>WER ↓</th> |
|
|
<th>Downloads</th> |
|
|
<th>Paper</th> |
|
|
</tr> |
|
|
</thead> |
|
|
<tbody> |
|
|
""" |
|
|
|
|
|
for _, row in formatted_df.iterrows(): |
|
|
html_table += f""" |
|
|
<tr> |
|
|
<td>{row['model']}</td> |
|
|
<td>{row['organization']}</td> |
|
|
<td>{row['type']}</td> |
|
|
<td>{row['task']}</td> |
|
|
<td>{row['chrf']}</td> |
|
|
<td>{row['cer']}</td> |
|
|
<td>{row['wer']}</td> |
|
|
<td>{row['downloads']}</td> |
|
|
<td>{row['paper']}</td> |
|
|
</tr> |
|
|
""" |
|
|
|
|
|
html_table += """ |
|
|
</tbody> |
|
|
</table> |
|
|
</div> |
|
|
<div class="footer"> |
|
|
<span>Showing {count} of {total} models</span> |
|
|
<div> |
|
|
<a href="https://github.com/mbzuai-oryx/KITAB-Bench" target="_blank">GitHub Repository</a> |
|
|
<span style="margin: 0 8px;">|</span> |
|
|
<a href="https://arxiv.org/abs/2502.14949" target="_blank">KITAB-Bench Paper</a> |
|
|
</div> |
|
|
</div> |
|
|
""".format(count=len(filtered_df), total=len(df_orig)) |
|
|
|
|
|
return html_table |
|
|
|
|
|
type_filter.change(update_table, [type_filter, search_input, sort_by, sort_order], table_output) |
|
|
search_input.change(update_table, [type_filter, search_input, sort_by, sort_order], table_output) |
|
|
sort_by.change(update_table, [type_filter, search_input, sort_by, sort_order], table_output) |
|
|
sort_order.change(update_table, [type_filter, search_input, sort_by, sort_order], table_output) |
|
|
|
|
|
demo.load(update_table, [type_filter, search_input, sort_by, sort_order], table_output) |
|
|
|
|
|
gr.HTML(""" |
|
|
<div style="margin-top: 20px; text-align: center; font-size: 0.8rem; color: #6B7280;"> |
|
|
<p>For more information about the KITAB-Bench, visit the <a href="https://mbzuai-oryx.github.io/KITAB-Bench/" target="_blank">project website</a>.</p> |
|
|
</div> |
|
|
""") |
|
|
|
|
|
return demo |
|
|
|
|
|
demo = create_leaderboard_interface() |
|
|
|
|
|
if __name__ == "__main__": |
|
|
demo.launch() |
