Upload 4 files

README.md

@@ -1,12 +1,20 @@
 ---
-title: Test
-emoji: 🌖
-colorFrom: yellow
-colorTo: green
+title: VBench Leaderboard
+emoji: 📊
+colorFrom: indigo
+colorTo: pink
 sdk: gradio
-sdk_version: 5.23.1
+sdk_version: 4.36.1
 app_file: app.py
 pinned: false
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# VBench Leaderboard
+
+## Space Description
+
+- **Repository:** [VBench](https://github.com/Vchitect/VBench)
+- **Paper:** [2311.17982](arxiv.org/abs/2311.17982)
+- **Paper:** [2411.13503](arxiv.org/abs/2411.13503)
+- **Point of Contact:** mailto:[Vchitect]([email protected])

app.py

@@ -0,0 +1,968 @@
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+import os
+import io
+import gradio as gr
+import pandas as pd
+import json
+import shutil
+import tempfile
+import datetime
+import zipfile
+import numpy as np
+
+
+from constants import *
+from huggingface_hub import Repository
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+global data_component, filter_component
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_link: str,
+    team_name: str,
+    contact_email: str,
+    access_type: str,
+    model_publish: str,
+    model_resolution: str,
+    model_fps: str,
+    model_frame: str,
+    model_video_length: str,
+    model_checkpoint: str,
+    model_commit_id: str,
+    model_video_format: str
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    if  model_link == '' or model_name_textbox == '' or contact_email == '':
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+    # upload_data=json.loads(input_file)
+    upload_content = input_file
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    
+    now = datetime.datetime.now()
+    update_time = now.strftime("%Y-%m-%d")  # Capture update time
+    with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
+        f.write(input_file)
+    # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
+
+    csv_data = pd.read_csv(CSV_DIR)
+
+    if revision_name_textbox == '':
+        col = csv_data.shape[0]
+        model_name = model_name_textbox.replace(',',' ')
+    else:
+        model_name = revision_name_textbox.replace(',',' ')
+        model_name_list = csv_data['Model Name (clickable)']
+        name_list = [name.split(']')[0][1:] for name in model_name_list]
+        if revision_name_textbox not in name_list:
+            col = csv_data.shape[0]
+        else:
+            col = name_list.index(revision_name_textbox)    
+    if model_link == '':
+        model_name = model_name  # no url
+    else:
+        model_name = '[' + model_name + '](' + model_link + ')'
+
+    os.makedirs(filename, exist_ok=True)
+    with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
+        zip_ref.extractall(filename)
+
+    upload_data = {}
+    for file in os.listdir(filename):
+        if file.startswith('.') or file.startswith('__'):
+            print(f"Skip the file: {file}")
+            continue
+        cur_file = os.path.join(filename, file)
+        if os.path.isdir(cur_file):
+            for subfile in os.listdir(cur_file):
+                if subfile.endswith(".json"):
+                    with open(os.path.join(cur_file, subfile)) as ff:
+                        cur_json = json.load(ff)
+                        print(file, type(cur_json))
+                        if isinstance(cur_json, dict):
+                            print(cur_json.keys())
+                            for key in cur_json:
+                                upload_data[key.replace('_',' ')] = cur_json[key][0]
+                                print(f"{key}:{cur_json[key][0]}")
+        elif cur_file.endswith('json'):
+            with open(cur_file) as ff:
+                cur_json = json.load(ff)
+                print(file, type(cur_json))
+                if isinstance(cur_json, dict):
+                    print(cur_json.keys())
+                    for key in cur_json:
+                        upload_data[key.replace('_',' ')] = cur_json[key][0]
+                        print(f"{key}:{cur_json[key][0]}")
+    # add new data
+    new_data = [model_name]
+    print('upload_data:', upload_data)
+    for key in TASK_INFO:
+        if key in upload_data:
+            new_data.append(upload_data[key])
+        else:
+            new_data.append(0)
+    if team_name =='' or 'vbench' in team_name.lower():
+        new_data.append("User Upload")
+    else:
+        new_data.append(team_name)
+
+    new_data.append(contact_email.replace(',',' and ')) # Add contact email [private]
+    new_data.append(update_time)  # Add the update time
+    new_data.append(team_name)
+    new_data.append(access_type)
+
+    csv_data.loc[col] = new_data
+    csv_data = csv_data.to_csv(CSV_DIR, index=False)
+    with open(INFO_DIR,'a') as f:
+        f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n")
+    submission_repo.push_to_hub()
+    print("success update", model_name)
+    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+
+def add_new_eval_i2v(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_link: str,
+    team_name: str,
+    contact_email: str,
+    access_type: str,
+    model_publish: str,
+    model_resolution: str,
+    model_fps: str,
+    model_frame: str,
+    model_video_length: str,
+    model_checkpoint: str,
+    model_commit_id: str,
+    model_video_format: str
+):
+    COLNAME2KEY={
+        "Video-Text Camera Motion":"camera_motion",
+        "Video-Image Subject Consistency": "i2v_subject",
+        "Video-Image Background Consistency": "i2v_background",
+        "Subject Consistency": "subject_consistency",
+        "Background Consistency": "background_consistency",
+        "Motion Smoothness": "motion_smoothness",
+        "Dynamic Degree": "dynamic_degree",
+        "Aesthetic Quality": "aesthetic_quality",
+        "Imaging Quality": "imaging_quality",
+        "Temporal Flickering": "temporal_flickering"
+        }
+    if input_file is None:
+        return "Error! Empty file!"
+    if  model_link == '' or model_name_textbox == '' or contact_email == '':
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+    
+    upload_content = input_file
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    
+    now = datetime.datetime.now()
+    update_time = now.strftime("%Y-%m-%d")  # Capture update time
+    with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
+        f.write(input_file)
+    # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
+
+    csv_data = pd.read_csv(I2V_DIR)
+
+    if revision_name_textbox == '':
+        col = csv_data.shape[0]
+        model_name = model_name_textbox.replace(',',' ')
+    else:
+        model_name = revision_name_textbox.replace(',',' ')
+        model_name_list = csv_data['Model Name (clickable)']
+        name_list = [name.split(']')[0][1:] for name in model_name_list]
+        if revision_name_textbox not in name_list:
+            col = csv_data.shape[0]
+        else:
+            col = name_list.index(revision_name_textbox)    
+    if model_link == '':
+        model_name = model_name  # no url
+    else:
+        model_name = '[' + model_name + '](' + model_link + ')'
+
+    os.makedirs(filename, exist_ok=True)
+    with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
+        zip_ref.extractall(filename)
+
+    upload_data = {}
+    for file in os.listdir(filename):
+        if file.startswith('.') or file.startswith('__'):
+            print(f"Skip the file: {file}")
+            continue
+        cur_file = os.path.join(filename, file)
+        if os.path.isdir(cur_file):
+            for subfile in os.listdir(cur_file):
+                if subfile.endswith(".json"):
+                    with open(os.path.join(cur_file, subfile)) as ff:
+                        cur_json = json.load(ff)
+                        print(file, type(cur_json))
+                        if isinstance(cur_json, dict):
+                            print(cur_json.keys())
+                            for key in cur_json:
+                                upload_data[key] = cur_json[key][0]
+                                print(f"{key}:{cur_json[key][0]}")
+        elif cur_file.endswith('json'):
+            with open(cur_file) as ff:
+                cur_json = json.load(ff)
+                print(file, type(cur_json))
+                if isinstance(cur_json, dict):
+                    print(cur_json.keys())
+                    for key in cur_json:
+                        upload_data[key] = cur_json[key][0]
+                        print(f"{key}:{cur_json[key][0]}")
+    # add new data
+    new_data = [model_name]
+    print('upload_data:', upload_data)
+    I2V_HEAD= ["Video-Text Camera Motion",
+    "Video-Image Subject Consistency",
+    "Video-Image Background Consistency",
+    "Subject Consistency",
+    "Background Consistency",
+    "Temporal Flickering",
+    "Motion Smoothness",
+    "Dynamic Degree",
+    "Aesthetic Quality",
+    "Imaging Quality" ]
+    for key in I2V_HEAD :
+        sub_key = COLNAME2KEY[key]
+        if sub_key in upload_data:
+            new_data.append(upload_data[sub_key])
+        else:
+            new_data.append(0)
+    if team_name =='' or 'vbench' in team_name.lower():
+        new_data.append("User Upload")
+    else:
+        new_data.append(team_name)
+
+    new_data.append(contact_email.replace(',',' and ')) # Add contact email [private]
+    new_data.append(update_time)  # Add the update time
+    new_data.append(team_name)
+    new_data.append(access_type)
+
+    csv_data.loc[col] = new_data
+    csv_data = csv_data.to_csv(I2V_DIR , index=False)
+    with open(INFO_DIR,'a') as f:
+        f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n")
+    submission_repo.push_to_hub()
+    print("success update", model_name)
+    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+
+def get_normalized_df(df):
+    # final_score = df.drop('name', axis=1).sum(axis=1)
+    # df.insert(1, 'Overall Score', final_score)
+    normalize_df = df.copy().fillna(0.0)
+    for column in normalize_df.columns[1:-5]:
+        min_val = NORMALIZE_DIC[column]['Min']
+        max_val = NORMALIZE_DIC[column]['Max']
+        normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
+    return normalize_df
+
+def get_normalized_i2v_df(df):
+    normalize_df = df.copy().fillna(0.0)
+    for column in normalize_df.columns[1:-5]:
+        min_val = NORMALIZE_DIC_I2V[column]['Min']
+        max_val = NORMALIZE_DIC_I2V[column]['Max']
+        normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
+    return normalize_df
+
+
+def calculate_selected_score(df, selected_columns):
+    # selected_score = df[selected_columns].sum(axis=1)
+    selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST]
+    selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST]
+    selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY])
+    selected_semantic_score = df[selected_SEMANTIC].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_SEMANTIC ])
+    if selected_quality_score.isna().any().any() and selected_semantic_score.isna().any().any():
+        selected_score =  (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
+        return selected_score.fillna(0.0)
+    if selected_quality_score.isna().any().any():
+        return selected_semantic_score
+    if selected_semantic_score.isna().any().any():
+        return selected_quality_score
+    # print(selected_semantic_score,selected_quality_score )
+    selected_score =  (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
+    return selected_score.fillna(0.0)
+
+def calculate_selected_score_i2v(df, selected_columns):
+    # selected_score = df[selected_columns].sum(axis=1)
+    selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST]
+    selected_I2V = [i for i in selected_columns if i in I2V_LIST]
+    selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY])
+    selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ])
+    if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any():
+        selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
+        return selected_score.fillna(0.0)
+    if selected_quality_score.isna().any().any():
+        return selected_i2v_score
+    if selected_i2v_score.isna().any().any():
+        return selected_quality_score
+    # print(selected_i2v_score,selected_quality_score )
+    selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
+    return selected_score.fillna(0.0)
+
+def get_final_score(df, selected_columns):
+    normalize_df = get_normalized_df(df)
+    #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
+    try:
+        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
+            normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
+    except:
+        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
+            normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
+    quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST])
+    semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ])
+    final_score =  (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
+    if 'Total Score' in df:
+        df['Total Score'] = final_score
+    else:
+        df.insert(1, 'Total Score', final_score)
+    if 'Semantic Score' in df:
+        df['Semantic Score'] = semantic_score
+    else:
+        df.insert(2, 'Semantic Score', semantic_score)
+    if 'Quality Score' in df:
+        df['Quality Score'] = quality_score
+    else:
+        df.insert(3, 'Quality Score', quality_score)
+    selected_score = calculate_selected_score(normalize_df, selected_columns)
+    if 'Selected Score' in df:
+        df['Selected Score'] = selected_score
+    else:
+        df.insert(1, 'Selected Score', selected_score)
+    return df
+
+def get_final_score_i2v(df, selected_columns):
+    normalize_df = get_normalized_i2v_df(df)
+    try:
+        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
+            normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
+    except:
+        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
+            normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
+    quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])
+    i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ])
+    final_score =  (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
+    if 'Total Score' in df:
+        df['Total Score'] = final_score
+    else:
+        df.insert(1, 'Total Score', final_score)
+    if 'I2V Score' in df:
+        df['I2V Score'] = i2v_score
+    else:
+        df.insert(2, 'I2V Score', i2v_score)
+    if 'Quality Score' in df:
+        df['Quality Score'] = quality_score
+    else:
+        df.insert(3, 'Quality Score', quality_score)
+    selected_score = calculate_selected_score_i2v(normalize_df, selected_columns)
+    if 'Selected Score' in df:
+        df['Selected Score'] = selected_score
+    else:
+        df.insert(1, 'Selected Score', selected_score)
+    # df.loc[df[9:].isnull().any(axis=1), ['Total Score', 'I2V Score']] = 'N.A.'
+    mask = df.iloc[:, 5:-5].isnull().any(axis=1)
+    df.loc[mask, ['Total Score', 'I2V Score','Selected Score' ]] = np.nan
+    # df.fillna('N.A.', inplace=True)
+    return df
+
+
+
+def get_final_score_quality(df, selected_columns):
+    normalize_df = get_normalized_df(df)
+    for name in normalize_df.drop('Model Name (clickable)', axis=1):
+        normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
+    quality_score = normalize_df[QUALITY_TAB].sum(axis=1) / sum([DIM_WEIGHT[i] for i in QUALITY_TAB])
+
+    if 'Quality Score' in df:
+        df['Quality Score'] = quality_score
+    else:
+        df.insert(1, 'Quality Score', quality_score)
+    # selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns)
+    selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns])
+    if 'Selected Score' in df:
+        df['Selected Score'] = selected_score
+    else:
+        df.insert(1, 'Selected Score', selected_score)
+    return df
+
+
+
+def get_baseline_df():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(CSV_DIR)
+    df = get_final_score(df, checkbox_group.value)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    # print(present_columns)
+    df = df[present_columns]
+    # Add this line to display the results evaluated by VBench by default
+    df = df[df['Evaluated by'] == 'VBench Team']
+    df = convert_scores_to_percentage(df)
+    return df
+
+def get_baseline_df_quality():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(QUALITY_DIR)
+    df = get_final_score_quality(df, checkbox_group_quality.value)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    present_columns = MODEL_INFO_TAB_QUALITY + checkbox_group_quality.value
+    df = df[present_columns]
+    df = convert_scores_to_percentage(df)
+    return df
+
+def get_baseline_df_i2v():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(I2V_DIR)
+    df = get_final_score_i2v(df, checkbox_group_i2v.value)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value
+    # df = df[df["Sampled by"] == 'VBench Team'] 
+    df = df[present_columns]
+    df = convert_scores_to_percentage(df)
+    return df
+
+def get_baseline_df_long():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(LONG_DIR)
+    df = get_final_score(df, checkbox_group.value)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    # df = df[df["Sampled by"] == 'VBench Team'] 
+    df = df[present_columns]
+    df = convert_scores_to_percentage(df)
+    return df
+
+def get_all_df(selected_columns, dir=CSV_DIR):
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(dir)
+    df = get_final_score(df, selected_columns)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    return df
+    
+def get_all_df_quality(selected_columns, dir=QUALITY_DIR):
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(dir)
+    df = get_final_score_quality(df, selected_columns)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    return df
+
+def get_all_df_i2v(selected_columns, dir=I2V_DIR):
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(dir)
+    df = get_final_score_i2v(df, selected_columns)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    return df
+
+def get_all_df_long(selected_columns, dir=LONG_DIR):
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(dir)
+    df = get_final_score(df, selected_columns)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    return df
+
+
+def convert_scores_to_percentage(df):
+    # Operate on every column in the DataFrame (except the'name 'column)
+    if "Sampled by" in df.columns:
+        skip_col =3
+    else:
+        skip_col =1
+    print(df)
+    for column in df.columns[skip_col:]:  # 假设第一列是'name'
+        # if df[column].isdigit():
+        # print(df[column])
+        # is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all()
+        valid_numeric_count = pd.to_numeric(df[column], errors='coerce').notna().sum()
+        if valid_numeric_count > 0:
+            df[column] = round(df[column] * 100,2)
+            df[column] = df[column].apply(lambda x: f"{x:05.2f}%" if pd.notna(pd.to_numeric(x, errors='coerce')) else x)
+            # df[column] = df[column].apply(lambda x:  f"{x:05.2f}") + '%'
+    return df
+
+def choose_all_quailty():
+    return gr.update(value=QUALITY_LIST)
+
+def choose_all_semantic():
+    return gr.update(value=SEMANTIC_LIST)
+
+def disable_all():
+    return gr.update(value=[])
+    
+def enable_all():
+    return gr.update(value=TASK_INFO)
+
+# select function
+def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbench_team_eval=False):
+    updated_data = get_all_df(selected_columns, CSV_DIR)
+    if vbench_team_sample:
+        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] 
+    if vbench_team_eval:
+        updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
+    #print(updated_data)
+    # columns:
+    selected_columns = [item for item in TASK_INFO if item in selected_columns]
+    present_columns = MODEL_INFO + selected_columns
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
+    updated_data = convert_scores_to_percentage(updated_data)
+    updated_headers = present_columns
+    print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE )
+    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+    # print(updated_data,present_columns,update_datatype)
+    filter_component = gr.components.Dataframe(
+        value=updated_data, 
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component#.value
+
+def on_filter_model_size_method_change_quality(selected_columns):
+    updated_data = get_all_df_quality(selected_columns, QUALITY_DIR)
+    #print(updated_data)
+    # columns:
+    selected_columns = [item for item in QUALITY_TAB if item in selected_columns]
+    present_columns = MODEL_INFO_TAB_QUALITY + selected_columns
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
+    updated_data = convert_scores_to_percentage(updated_data)
+    updated_headers = present_columns
+    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+    # print(updated_data,present_columns,update_datatype)
+    filter_component = gr.components.Dataframe(
+        value=updated_data, 
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component#.value
+
+def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample, vbench_team_eval=False):
+    updated_data = get_all_df_i2v(selected_columns, I2V_DIR)
+    if vbench_team_sample:
+        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
+    # if vbench_team_eval:
+    #     updated_data = updated_data[updated_data['Eval'] == 'VBench Team']
+    selected_columns = [item for item in I2V_TAB if item in selected_columns]
+    present_columns = MODEL_INFO_TAB_I2V + selected_columns
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
+    updated_data = convert_scores_to_percentage(updated_data)
+    updated_headers = present_columns
+    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers]
+    # print(updated_data,present_columns,update_datatype)
+    filter_component = gr.components.Dataframe(
+        value=updated_data,
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component#.value
+
+def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample, vbench_team_eval=False):
+    updated_data = get_all_df_long(selected_columns, LONG_DIR)
+    if vbench_team_sample:
+        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
+    if vbench_team_eval:
+        updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
+    selected_columns = [item for item in TASK_INFO if item in selected_columns]
+    present_columns = MODEL_INFO + selected_columns
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
+    updated_data = convert_scores_to_percentage(updated_data)
+    updated_headers = present_columns
+    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+    filter_component = gr.components.Dataframe(
+        value=updated_data, 
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component#.value
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # Table 0
+        with gr.TabItem("📊 VBench", elem_id="vbench-tab-table", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=14,
+                    )
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+            with gr.Row():
+                with gr.Column(scale=0.2):
+                    choosen_q = gr.Button("Select Quality Dimensions")
+                    choosen_s = gr.Button("Select Semantic Dimensions")
+                    # enable_b = gr.Button("Select All")
+                    disable_b = gr.Button("Deselect All")
+
+                with gr.Column(scale=0.8):
+                    vbench_team_filter = gr.Checkbox(
+                            label="Sampled by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                    vbench_validate_filter = gr.Checkbox(
+                        label="Evaluated by VBench Team (Uncheck to view all submissions)",
+                        value=True,
+                        interactive=True
+                    )
+                    # selection for column part:
+                    checkbox_group = gr.CheckboxGroup(
+                        choices=TASK_INFO,
+                        value=DEFAULT_INFO,
+                        label="Evaluation Dimension",
+                        interactive=True,
+                    )
+
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                height=700,
+                )
+    
+            choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
+            choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
+            # enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
+            disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+            vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+            vbench_validate_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
+        # Table 1
+        with gr.TabItem("Video Quality", elem_id="vbench-tab-table", id=2):
+            with gr.Accordion("INSTRUCTION", open=False):
+                    citation_button = gr.Textbox(
+                        value=QUALITY_CLAIM_TEXT,
+                        label="",
+                        elem_id="quality-button",
+                        lines=2,
+                    )
+            with gr.Row():
+                with gr.Column(scale=1.0):
+                    # selection for column part:
+                    
+                    checkbox_group_quality = gr.CheckboxGroup(
+                        choices=QUALITY_TAB,
+                        value=QUALITY_TAB,
+                        label="Evaluation Quality Dimension",
+                        interactive=True,
+                    )
+
+            data_component_quality = gr.components.Dataframe(
+                value=get_baseline_df_quality, 
+                headers=COLUMN_NAMES_QUALITY,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality)
+        
+        # Table i2v
+        with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=3):
+            with gr.Accordion("NOTE", open=False):
+                    i2v_note_button = gr.Textbox(
+                        value=I2V_CLAIM_TEXT,
+                        label="",
+                        elem_id="quality-button",
+                        lines=3,
+                    )
+            with gr.Row():
+                with gr.Column(scale=1.0):
+                    # selection for column part:
+                    with gr.Row():
+                        vbench_team_filter_i2v = gr.Checkbox(
+                            label="Sampled by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                        vbench_validate_filter_i2v = gr.Checkbox(
+                                label="Evaluated by VBench Team (Uncheck to view all submissions)",
+                                value=False,
+                                interactive=True
+                            )
+                    checkbox_group_i2v = gr.CheckboxGroup(
+                        choices=I2V_TAB,
+                        value=I2V_TAB,
+                        label="Evaluation Quality Dimension",
+                        interactive=True,
+                    )
+
+            data_component_i2v = gr.components.Dataframe(
+                value=get_baseline_df_i2v, 
+                headers=COLUMN_NAMES_I2V,
+                type="pandas", 
+                datatype=I2V_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
+            vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
+            vbench_validate_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
+        
+        with gr.TabItem("📊 VBench-Long", elem_id="vbench-tab-table", id=4):
+            with gr.Row():
+                with gr.Accordion("INSTRUCTION", open=False):
+                    citation_button = gr.Textbox(
+                        value=LONG_CLAIM_TEXT,
+                        label="",
+                        elem_id="long-ins-button",
+                        lines=2,
+                    )
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+            with gr.Row():
+                with gr.Column(scale=0.2):
+                    choosen_q_long = gr.Button("Select Quality Dimensions")
+                    choosen_s_long = gr.Button("Select Semantic Dimensions")
+                    enable_b_long = gr.Button("Select All")
+                    disable_b_long = gr.Button("Deselect All")
+
+                with gr.Column(scale=0.8):
+                    with gr.Row():
+                        vbench_team_filter_long = gr.Checkbox(
+                            label="Sampled by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                        vbench_validate_filter_long = gr.Checkbox(
+                            label="Evaluated by VBench Team (Uncheck to view all submissions)",
+                            value=False,
+                            interactive=True
+                        )
+                    checkbox_group_long = gr.CheckboxGroup(
+                        choices=TASK_INFO,
+                        value=DEFAULT_INFO,
+                        label="Evaluation Dimension",
+                        interactive=True,
+                    )
+
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df_long, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                height=700,
+                )
+    
+            choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
+            checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
+            vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
+            vbench_validate_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
+
+        # table info
+        with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=5):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+        
+        # table submission 
+        with gr.TabItem("🚀 [T2V]Submit here! ", elem_id="mvbench-tab-table", id=6):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("Here is a required field", elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="Required field"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
+                    )
+                    access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
+
+                with gr.Column():
+                    model_link = gr.Textbox(
+                        label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
+                    )
+                    team_name = gr.Textbox(
+                        label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
+                    )
+                    contact_email = gr.Textbox(
+                        label="E-Mail(Will not be displayed)", placeholder="Required field"
+                    )
+            with gr.Row():
+                gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
+            with gr.Row():
+                    release_time = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
+                    model_resolution = gr.Textbox(label="resolution", placeholder="Width x Height")
+                    model_fps = gr.Textbox(label="model fps", placeholder="FPS(int)")
+                    model_frame = gr.Textbox(label="model frame count", placeholder="INT")
+                    model_video_length = gr.Textbox(label="model video length", placeholder="float(2.0)")
+                    model_checkpoint = gr.Textbox(label="model checkpoint", placeholder="optional")
+                    model_commit_id = gr.Textbox(label="github commit id", placeholder='main')
+                    model_video_format = gr.Textbox(label="pipeline format", placeholder='mp4')
+            with gr.Column():
+                input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+                submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
+                fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
+                
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs = [
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_link,
+                        team_name,
+                        contact_email,
+                        release_time,
+                        access_type,
+                        model_resolution,
+                        model_fps,
+                        model_frame,
+                        model_video_length,
+                        model_checkpoint,
+                        model_commit_id,
+                        model_video_format
+                    ],
+                    outputs=[submit_button, submit_succ_button, fail_textbox]
+                )
+                
+        with gr.TabItem("🚀 [I2V]Submit here! ", elem_id="mvbench-i2v-tab-table", id=7):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your i2v model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("Here is a required field", elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox_i2v = gr.Textbox(
+                        label="Model name", placeholder="Required field"
+                        )
+                    revision_name_textbox_i2v = gr.Textbox(
+                        label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
+                    )
+                    access_type_i2v = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
+
+
+                with gr.Column():
+                    model_link_i2v = gr.Textbox(
+                        label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
+                    )
+                    team_name_i2v = gr.Textbox(
+                        label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
+                    )
+                    contact_email_i2v = gr.Textbox(
+                        label="E-Mail(Will not be displayed)", placeholder="Required field"
+                    )
+            with gr.Row():
+                gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
+            with gr.Row():
+                    release_time_i2v = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
+                    model_resolution_i2v = gr.Textbox(label="resolution", placeholder="Width x Height")
+                    model_fps_i2v = gr.Textbox(label="model fps", placeholder="FPS(int)")
+                    model_frame_i2v = gr.Textbox(label="model frame count", placeholder="INT")
+                    model_video_length_i2v = gr.Textbox(label="model video length", placeholder="float(2.0)")
+                    model_checkpoint_i2v = gr.Textbox(label="model checkpoint", placeholder="optional")
+                    model_commit_id_i2v = gr.Textbox(label="github commit id", placeholder='main')
+                    model_video_format_i2v = gr.Textbox(label="pipeline format", placeholder='mp4')
+            with gr.Column():
+                input_file_i2v = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
+                submit_button_i2v = gr.Button("Submit Eval")
+                submit_succ_button_i2v = gr.Markdown("Submit Success! Please press refresh and retfurn to LeaderBoard!", visible=False)
+                fail_textbox_i2v = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
+                
+    
+                submission_result_i2v = gr.Markdown()
+                submit_button_i2v.click(
+                    add_new_eval_i2v,
+                    inputs = [
+                        input_file_i2v,
+                        model_name_textbox_i2v,
+                        revision_name_textbox_i2v,
+                        model_link_i2v,
+                        team_name_i2v,
+                        contact_email_i2v,
+                        release_time_i2v,
+                        access_type_i2v,
+                        model_resolution_i2v,
+                        model_fps_i2v,
+                        model_frame_i2v,
+                        model_video_length_i2v,
+                        model_checkpoint_i2v,
+                        model_commit_id_i2v,
+                        model_video_format_i2v
+                    ],
+                    outputs=[submit_button_i2v, submit_succ_button_i2v, fail_textbox_i2v]
+                )
+
+
+
+    def refresh_data():
+        value1 = get_baseline_df()
+        return value1
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component)
+
+
+block.launch()

constants.py

@@ -0,0 +1,266 @@
+import os
+# this is .py for store constants 
+MODEL_INFO = [
+    "Model Name (clickable)",
+    "Sampled by",
+    "Evaluated by",
+    "Accessibility",
+    "Date",
+    "Total Score",
+    "Quality Score",
+    "Semantic Score",
+    "Selected Score",
+    ]
+
+MODEL_INFO_TAB_QUALITY = [
+    "Model Name (clickable)",
+    "Quality Score",
+    "Selected Score"
+]
+
+MODEL_INFO_TAB_I2V = [
+    "Model Name (clickable)",
+    "Sampled by",
+    "Evaluated by",
+    "Accessibility",
+    "Date",
+    "Total Score",
+    "I2V Score",
+    "Quality Score",
+    "Selected Score"
+]
+
+TASK_INFO = [
+    "subject consistency",
+    "background consistency",
+    "temporal flickering",
+    "motion smoothness",
+    "dynamic degree",
+    "aesthetic quality",
+    "imaging quality",
+    "object class",
+    "multiple objects",
+    "human action",
+    "color",
+    "spatial relationship",
+    "scene",
+    "appearance style",
+    "temporal style",
+    "overall consistency"
+]
+
+DEFAULT_INFO = [
+    "subject consistency",
+    "background consistency",
+    "temporal flickering",
+    "motion smoothness",
+    "dynamic degree",
+    "aesthetic quality",
+    "imaging quality",
+    "object class",
+    "multiple objects",
+    "human action",
+    "color",
+    "spatial relationship",
+    "scene",
+    "appearance style",
+    "temporal style",
+    "overall consistency"
+    ]
+
+QUALITY_LIST = [ 
+    "subject consistency",
+    "background consistency",
+    "temporal flickering",
+    "motion smoothness",
+    "aesthetic quality",
+    "imaging quality",
+    "dynamic degree",]
+
+SEMANTIC_LIST = [
+    "object class",
+    "multiple objects",
+    "human action",
+    "color",
+    "spatial relationship",
+    "scene",
+    "appearance style",
+    "temporal style",
+    "overall consistency"
+]
+
+QUALITY_TAB = [ 
+    "subject consistency",
+    "background consistency",
+    "motion smoothness",
+    "aesthetic quality",
+    "imaging quality",
+    "dynamic degree",]
+
+I2V_LIST = [
+    "Video-Text Camera Motion",
+    "Video-Image Subject Consistency",
+    "Video-Image Background Consistency",
+]
+
+I2V_QUALITY_LIST = [
+    "Subject Consistency",
+    "Background Consistency",
+    "Motion Smoothness",
+    "Dynamic Degree",
+    "Aesthetic Quality",
+    "Imaging Quality",
+    # "Temporal Flickering"
+]
+
+I2V_TAB = [
+    "Video-Text Camera Motion",
+    "Video-Image Subject Consistency",
+    "Video-Image Background Consistency",
+    "Subject Consistency",
+    "Background Consistency",
+    "Motion Smoothness",
+    "Dynamic Degree",
+    "Aesthetic Quality",
+    "Imaging Quality",
+    # "Temporal Flickering"
+]
+
+DIM_WEIGHT = {
+"subject consistency":1,
+"background consistency":1,
+"temporal flickering":1,
+"motion smoothness":1,
+"aesthetic quality":1,
+"imaging quality":1,
+"dynamic degree":0.5,
+"object class":1,
+"multiple objects":1,
+"human action":1,
+"color":1,
+"spatial relationship":1,
+"scene":1,
+"appearance style":1,
+"temporal style":1,
+"overall consistency":1
+}
+
+DIM_WEIGHT_I2V = {
+"Video-Text Camera Motion": 0.1,
+"Video-Image Subject Consistency": 1,
+"Video-Image Background Consistency": 1,
+"Subject Consistency": 1,
+"Background Consistency": 1,
+"Motion Smoothness": 1,
+"Dynamic Degree": 0.5,
+"Aesthetic Quality": 1,
+"Imaging Quality": 1,
+"Temporal Flickering": 1
+}
+
+SEMANTIC_WEIGHT = 1
+QUALITY_WEIGHT = 4
+I2V_WEIGHT = 1.0
+I2V_QUALITY_WEIGHT = 1.0
+
+DATA_TITILE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
+I2V_TITILE_TYPE =  ['markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
+
+SUBMISSION_NAME = "vbench_leaderboard_submission"
+SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/Vchitect/", SUBMISSION_NAME)
+CSV_DIR = "./vbench_leaderboard_submission/results.csv"
+QUALITY_DIR = "./vbench_leaderboard_submission/quality.csv"
+I2V_DIR = "./vbench_leaderboard_submission/i2v_results.csv"
+LONG_DIR = "./vbench_leaderboard_submission/long_debug.csv"
+INFO_DIR = "./vbench_leaderboard_submission/model_info.csv"
+
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+COLUMN_NAMES_QUALITY = MODEL_INFO_TAB_QUALITY + QUALITY_TAB
+COLUMN_NAMES_I2V = MODEL_INFO_TAB_I2V + I2V_TAB
+
+LEADERBORAD_INTRODUCTION = """# VBench Leaderboard
+    
+    *"Which Video Generation Model is better?"*  
+    🏆 Welcome to the leaderboard of the **VBench**! 🎦 *A Comprehensive Benchmark Suite for Video Generative Models* (**CVPR 2024 Spotlight**)   [![Code](https://img.shields.io/github/stars/Vchitect/VBench.svg?style=social&label=Official)](https://github.com/Vchitect/VBench) 
+    <div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
+    <a href='https://arxiv.org/abs/2311.17982'><img src='https://img.shields.io/badge/cs.CV-Paper-b31b1b?logo=arxiv&logoColor=red'></a>
+    <a href='https://vchitect.github.io/VBench-project/'><img src='https://img.shields.io/badge/VBench-Website-green?logo=googlechrome&logoColor=green'></a>
+    <a href='https://pypi.org/project/vbench/'><img src='https://img.shields.io/pypi/v/vbench'></a>
+    <a href='https://www.youtube.com/watch?v=7IhCC8Qqn8Y'><img src='https://img.shields.io/badge/YouTube-Video-c4302b?logo=youtube&logoColor=red'></a>
+    <a href='https://hits.seeyoufarm.com'><img src='https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2FVchitect%2FVBench&count_bg=%23FFA500&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=visitors&edge_flat=false'></a>
+    </div>
+    
+    - **Comprehensive Dimensions:** We carefully decompose video generation quality into 16 comprehensive dimensions to reveal individual model's strengths and weaknesses.
+    - **Human Alignment:** We conducted extensive experiments and human annotations to validate robustness of VBench.
+    - **Valuable Insights:** VBench provides multi-perspective insights useful for the community.  
+    
+    **Join Leaderboard**: Please see the [instructions](https://github.com/Vchitect/VBench/tree/master?tab=readme-ov-file#trophy-leaderboard) for 3 options to participate. One option is to follow [VBench Usage info](https://github.com/Vchitect/VBench?tab=readme-ov-file#usage), and upload the generated `result.json` file here. After clicking the `Submit here!` button, click the `Refresh` button.
+
+    **Model Information**: What are the details of these Video Generation Models? See [HERE](https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)
+    
+    **Credits**: This leaderboard is updated and maintained by the team of [VBench Contributors](https://github.com/Vchitect/VBench?tab=readme-ov-file#muscle-vbench-contributors).
+    """
+
+SUBMIT_INTRODUCTION = """# Submit on VBench Benchmark Introduction
+
+## 🎈
+1. Please note that you need to obtain the file `evaluation_results/*.json` by running VBench in Github. You may conduct an [Offline Check](https://github.com/Vchitect/VBench?tab=readme-ov-file#get-final-score-and-submit-to-leaderboard) before uploading.
+2. Then, pack these JSON files into a `ZIP` archive, ensuring that the top-level directory of the ZIP contains the individual JSON files. 
+3. Finally, upload the ZIP archive below.
+
+⚠️ Uploading generated videos or images of the model is invalid!
+⚠️ Submissions that do not correctly fill in the model name and model link may be deleted by the VBench team. The contact information you filled in will not be made public. 
+"""
+
+TABLE_INTRODUCTION = """
+    """
+
+LEADERBORAD_INFO = """
+       VBench, a comprehensive benchmark suite for video generative models. We design a comprehensive and hierarchical Evaluation Dimension Suite to decompose "video generation quality" into multiple well-defined dimensions to facilitate fine-grained and objective evaluation. For each dimension and each content category, we carefully design a Prompt Suite as test cases, and sample Generated Videos from a set of video generation models. For each evaluation dimension, we specifically design an Evaluation Method Suite, which uses carefully crafted method or designated pipeline for automatic objective evaluation. We also conduct Human Preference Annotation for the generated videos for each dimension, and show that VBench evaluation results are well aligned with human perceptions. VBench can provide valuable insights from multiple perspectives.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@inproceedings{huang2023vbench,
+     title={{VBench}: Comprehensive Benchmark Suite for Video Generative Models},
+     author={Huang, Ziqi and He, Yinan and Yu, Jiashuo and Zhang, Fan and Si, Chenyang and Jiang, Yuming and Zhang, Yuanhan and Wu, Tianxing and Jin, Qingyang and Chanpaisit, Nattapol and Wang, Yaohui and Chen, Xinyuan and Wang, Limin and Lin, Dahua and Qiao, Yu and Liu, Ziwei},
+     booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
+     year={2024}
+}"""
+
+QUALITY_CLAIM_TEXT = "We use all the videos on Sora website (https://openai.com/sora) for a preliminary evaluation, including the failure case videos Sora provided."
+
+I2V_CLAIM_TEXT = "Since the open-sourced SVD models do not accept text input during the I2V stage, we are unable to evaluate its `camera motion` in terms of `video-text consistency`. The total score is calculated based on all dimensions except `camera motion`."
+
+LONG_CLAIM_TEXT = "" 
+
+NORMALIZE_DIC = {
+  "subject consistency": {"Min": 0.1462, "Max": 1.0},
+  "background consistency": {"Min": 0.2615, "Max": 1.0},
+  "temporal flickering": {"Min": 0.6293, "Max": 1.0},
+  "motion smoothness": {"Min": 0.706, "Max": 0.9975},
+  "dynamic degree": {"Min": 0.0, "Max": 1.0},
+  "aesthetic quality": {"Min": 0.0, "Max": 1.0},
+  "imaging quality": {"Min": 0.0, "Max": 1.0},
+  "object class": {"Min": 0.0, "Max": 1.0},
+  "multiple objects": {"Min": 0.0, "Max": 1.0},
+  "human action": {"Min": 0.0, "Max": 1.0},
+  "color": {"Min": 0.0, "Max": 1.0},
+  "spatial relationship": {"Min": 0.0, "Max": 1.0},
+  "scene": {"Min": 0.0, "Max": 0.8222},
+  "appearance style": {"Min": 0.0009, "Max": 0.2855},
+  "temporal style": {"Min": 0.0, "Max": 0.364},
+  "overall consistency": {"Min": 0.0, "Max": 0.364}
+}
+
+NORMALIZE_DIC_I2V = {
+    "Video-Text Camera Motion" :{"Min": 0.0, "Max":1.0 },
+    "Video-Image Subject Consistency":{"Min": 0.1462, "Max": 1.0},
+    "Video-Image Background Consistency":{"Min": 0.2615, "Max":1.0 },
+    "Subject Consistency":{"Min": 0.1462, "Max": 1.0},
+    "Background Consistency":{"Min": 0.2615, "Max": 1.0 },
+    "Motion Smoothness":{"Min": 0.7060, "Max": 0.9975},
+    "Dynamic Degree":{"Min": 0.0, "Max": 1.0},
+    "Aesthetic Quality":{"Min": 0.0, "Max": 1.0},
+    "Imaging Quality":{"Min": 0.0, "Max": 1.0},
+    "Temporal Flickering":{"Min":0.6293, "Max": 1.0}
+}

requirements.txt

@@ -0,0 +1,3 @@
+gradio==3.23.0
+numpy
+pandas