Upload 2 files

app.py

@@ -259,356 +259,25 @@ def upload_file(files):
 #     print("success update", model_name)
 #     return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
 
-def get_normalized_df(df):
-    # final_score = df.drop('name', axis=1).sum(axis=1)
-    # df.insert(1, 'Overall Score', final_score)
-    normalize_df = df.copy().fillna(0.0)
-    for column in normalize_df.columns[1:-5]:
-        min_val = NORMALIZE_DIC[column]['Min']
-        max_val = NORMALIZE_DIC[column]['Max']
-        normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
-    return normalize_df
-
-def get_normalized_i2v_df(df):
-    normalize_df = df.copy().fillna(0.0)
-    for column in normalize_df.columns[1:-5]:
-        min_val = NORMALIZE_DIC_I2V[column]['Min']
-        max_val = NORMALIZE_DIC_I2V[column]['Max']
-        normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
-    return normalize_df
-
-
-def calculate_selected_score(df, selected_columns):
-    # selected_score = df[selected_columns].sum(axis=1)
-    selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST]
-    selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST]
-    selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY])
-    selected_semantic_score = df[selected_SEMANTIC].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_SEMANTIC ])
-    if selected_quality_score.isna().any().any() and selected_semantic_score.isna().any().any():
-        selected_score =  (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
-        return selected_score.fillna(0.0)
-    if selected_quality_score.isna().any().any():
-        return selected_semantic_score
-    if selected_semantic_score.isna().any().any():
-        return selected_quality_score
-    # print(selected_semantic_score,selected_quality_score )
-    selected_score =  (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
-    return selected_score.fillna(0.0)
-
-def calculate_selected_score_i2v(df, selected_columns):
-    # selected_score = df[selected_columns].sum(axis=1)
-    selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST]
-    selected_I2V = [i for i in selected_columns if i in I2V_LIST]
-    selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY])
-    selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ])
-    if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any():
-        selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
-        return selected_score.fillna(0.0)
-    if selected_quality_score.isna().any().any():
-        return selected_i2v_score
-    if selected_i2v_score.isna().any().any():
-        return selected_quality_score
-    # print(selected_i2v_score,selected_quality_score )
-    selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
-    return selected_score.fillna(0.0)
-
-def get_final_score(df, selected_columns):
-    normalize_df = get_normalized_df(df)
-    #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
-    try:
-        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
-            normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
-    except:
-        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
-            normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
-    quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST])
-    semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ])
-    final_score =  (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
-    if 'Total Score' in df:
-        df['Total Score'] = final_score
-    else:
-        df.insert(1, 'Total Score', final_score)
-    if 'Semantic Score' in df:
-        df['Semantic Score'] = semantic_score
-    else:
-        df.insert(2, 'Semantic Score', semantic_score)
-    if 'Quality Score' in df:
-        df['Quality Score'] = quality_score
-    else:
-        df.insert(3, 'Quality Score', quality_score)
-    selected_score = calculate_selected_score(normalize_df, selected_columns)
-    if 'Selected Score' in df:
-        df['Selected Score'] = selected_score
-    else:
-        df.insert(1, 'Selected Score', selected_score)
-    return df
-
-def get_final_score_i2v(df, selected_columns):
-    normalize_df = get_normalized_i2v_df(df)
-    try:
-        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1):
-            normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
-    except:
-        for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1):
-            normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
-    quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])
-    i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ])
-    final_score =  (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
-    if 'Total Score' in df:
-        df['Total Score'] = final_score
-    else:
-        df.insert(1, 'Total Score', final_score)
-    if 'I2V Score' in df:
-        df['I2V Score'] = i2v_score
-    else:
-        df.insert(2, 'I2V Score', i2v_score)
-    if 'Quality Score' in df:
-        df['Quality Score'] = quality_score
-    else:
-        df.insert(3, 'Quality Score', quality_score)
-    selected_score = calculate_selected_score_i2v(normalize_df, selected_columns)
-    if 'Selected Score' in df:
-        df['Selected Score'] = selected_score
-    else:
-        df.insert(1, 'Selected Score', selected_score)
-    # df.loc[df[9:].isnull().any(axis=1), ['Total Score', 'I2V Score']] = 'N.A.'
-    mask = df.iloc[:, 5:-5].isnull().any(axis=1)
-    df.loc[mask, ['Total Score', 'I2V Score','Selected Score' ]] = np.nan
-    # df.fillna('N.A.', inplace=True)
-    return df
-
-
-
-def get_final_score_quality(df, selected_columns):
-    normalize_df = get_normalized_df(df)
-    for name in normalize_df.drop('Model Name (clickable)', axis=1):
-        normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name]
-    quality_score = normalize_df[QUALITY_TAB].sum(axis=1) / sum([DIM_WEIGHT[i] for i in QUALITY_TAB])
-
-    if 'Quality Score' in df:
-        df['Quality Score'] = quality_score
-    else:
-        df.insert(1, 'Quality Score', quality_score)
-    # selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns)
-    selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns])
-    if 'Selected Score' in df:
-        df['Selected Score'] = selected_score
-    else:
-        df.insert(1, 'Selected Score', selected_score)
-    return df
-
-
-
 def get_baseline_df():
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
     df = pd.read_csv(CSV_DIR)
-    df = get_final_score(df, checkbox_group.value)
-    df = df.sort_values(by="Selected Score", ascending=False)
-    present_columns = MODEL_INFO + checkbox_group.value
-    # print(present_columns)
-    df = df[present_columns]
-    # Add this line to display the results evaluated by VBench by default
-    df = df[df['Evaluated by'] == 'VBench Team']
-    df = convert_scores_to_percentage(df)
-    return df
-
-def get_baseline_df_quality():
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(QUALITY_DIR)
-    df = get_final_score_quality(df, checkbox_group_quality.value)
-    df = df.sort_values(by="Selected Score", ascending=False)
-    present_columns = MODEL_INFO_TAB_QUALITY + checkbox_group_quality.value
-    df = df[present_columns]
-    df = convert_scores_to_percentage(df)
-    return df
-
-def get_baseline_df_i2v():
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(I2V_DIR)
-    df = get_final_score_i2v(df, checkbox_group_i2v.value)
-    df = df.sort_values(by="Selected Score", ascending=False)
-    present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value
-    # df = df[df["Sampled by"] == 'VBench Team'] 
-    df = df[present_columns]
-    df = convert_scores_to_percentage(df)
-    return df
-
-def get_baseline_df_long():
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(LONG_DIR)
-    df = get_final_score(df, checkbox_group.value)
-    df = df.sort_values(by="Selected Score", ascending=False)
-    present_columns = MODEL_INFO + checkbox_group.value
-    # df = df[df["Sampled by"] == 'VBench Team'] 
-    df = df[present_columns]
-    df = convert_scores_to_percentage(df)
-    return df
-
-def get_all_df(selected_columns, dir=CSV_DIR):
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(dir)
-    df = get_final_score(df, selected_columns)
-    df = df.sort_values(by="Selected Score", ascending=False)
-    return df
-    
-def get_all_df_quality(selected_columns, dir=QUALITY_DIR):
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(dir)
-    df = get_final_score_quality(df, selected_columns)
-    df = df.sort_values(by="Selected Score", ascending=False)
-    return df
+    df = df.sort_values(by=DEFAULT_INFO[0], ascending=False)
 
-def get_all_df_i2v(selected_columns, dir=I2V_DIR):
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(dir)
-    df = get_final_score_i2v(df, selected_columns)
-    df = df.sort_values(by="Selected Score", ascending=False)
+    # Add this line to display the results of all model types by default
+    df = df[df['Model Type'].isin(model_type_filter.value)]
+    # Add this line to display the results of both abilities by default
+    df = df[df['Ability'].isin(ability_filter.value)]    
     return df
 
-def get_all_df_long(selected_columns, dir=LONG_DIR):
+def get_all_df(dir=CSV_DIR):
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
     df = pd.read_csv(dir)
-    df = get_final_score(df, selected_columns)
-    df = df.sort_values(by="Selected Score", ascending=False)
     return df
 
 
-def convert_scores_to_percentage(df):
-    # Operate on every column in the DataFrame (except the'name 'column)
-    if "Sampled by" in df.columns:
-        skip_col =3
-    else:
-        skip_col =1
-    print(df)
-    for column in df.columns[skip_col:]:  # 假设第一列是'name'
-        # if df[column].isdigit():
-        # print(df[column])
-        # is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all()
-        valid_numeric_count = pd.to_numeric(df[column], errors='coerce').notna().sum()
-        if valid_numeric_count > 0:
-            df[column] = round(df[column] * 100,2)
-            df[column] = df[column].apply(lambda x: f"{x:05.2f}%" if pd.notna(pd.to_numeric(x, errors='coerce')) else x)
-            # df[column] = df[column].apply(lambda x:  f"{x:05.2f}") + '%'
-    return df
-
-def choose_all_quailty():
-    return gr.update(value=QUALITY_LIST)
-
-def choose_all_semantic():
-    return gr.update(value=SEMANTIC_LIST)
-
-def disable_all():
-    return gr.update(value=[])
-    
-def enable_all():
-    return gr.update(value=TASK_INFO)
-
-# select function
-def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbench_team_eval=False):
-    updated_data = get_all_df(selected_columns, CSV_DIR)
-    if vbench_team_sample:
-        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] 
-    if vbench_team_eval:
-        updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
-    #print(updated_data)
-    # columns:
-    selected_columns = [item for item in TASK_INFO if item in selected_columns]
-    present_columns = MODEL_INFO + selected_columns
-    updated_data = updated_data[present_columns]
-    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
-    updated_data = convert_scores_to_percentage(updated_data)
-    updated_headers = present_columns
-    print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE )
-    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
-    # print(updated_data,present_columns,update_datatype)
-    filter_component = gr.components.Dataframe(
-        value=updated_data, 
-        headers=updated_headers,
-        type="pandas", 
-        datatype=update_datatype,
-        interactive=False,
-        visible=True,
-        )
-    return filter_component#.value
-
-def on_filter_model_size_method_change_quality(selected_columns):
-    updated_data = get_all_df_quality(selected_columns, QUALITY_DIR)
-    #print(updated_data)
-    # columns:
-    selected_columns = [item for item in QUALITY_TAB if item in selected_columns]
-    present_columns = MODEL_INFO_TAB_QUALITY + selected_columns
-    updated_data = updated_data[present_columns]
-    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
-    updated_data = convert_scores_to_percentage(updated_data)
-    updated_headers = present_columns
-    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
-    # print(updated_data,present_columns,update_datatype)
-    filter_component = gr.components.Dataframe(
-        value=updated_data, 
-        headers=updated_headers,
-        type="pandas", 
-        datatype=update_datatype,
-        interactive=False,
-        visible=True,
-        )
-    return filter_component#.value
-
-def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample, vbench_team_eval=False):
-    updated_data = get_all_df_i2v(selected_columns, I2V_DIR)
-    if vbench_team_sample:
-        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
-    # if vbench_team_eval:
-    #     updated_data = updated_data[updated_data['Eval'] == 'VBench Team']
-    selected_columns = [item for item in I2V_TAB if item in selected_columns]
-    present_columns = MODEL_INFO_TAB_I2V + selected_columns
-    updated_data = updated_data[present_columns]
-    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
-    updated_data = convert_scores_to_percentage(updated_data)
-    updated_headers = present_columns
-    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers]
-    # print(updated_data,present_columns,update_datatype)
-    filter_component = gr.components.Dataframe(
-        value=updated_data,
-        headers=updated_headers,
-        type="pandas", 
-        datatype=update_datatype,
-        interactive=False,
-        visible=True,
-        )
-    return filter_component#.value
-
-def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample, vbench_team_eval=False):
-    updated_data = get_all_df_long(selected_columns, LONG_DIR)
-    if vbench_team_sample:
-        updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team']
-    if vbench_team_eval:
-        updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team']
-    selected_columns = [item for item in TASK_INFO if item in selected_columns]
-    present_columns = MODEL_INFO + selected_columns
-    updated_data = updated_data[present_columns]
-    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
-    updated_data = convert_scores_to_percentage(updated_data)
-    updated_headers = present_columns
-    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
-    filter_component = gr.components.Dataframe(
-        value=updated_data, 
-        headers=updated_headers,
-        type="pandas", 
-        datatype=update_datatype,
-        interactive=False,
-        visible=True,
-        )
-    return filter_component#.value
-
-
 block = gr.Blocks()
 with block:
     gr.Markdown(
@@ -624,238 +293,50 @@ with block:
                     label="Model Type",
                     interactive=True
                 )
-                # selection for column part:
-                checkbox_group = gr.CheckboxGroup(
-                    choices=TASK_INFO,
-                    value=DEFAULT_INFO,
-                    label="Evaluation Dimension",
-                    interactive=True,
+                ability_filter = gr.CheckboxGroup(
+                    choices=ABILITY,
+                    value=DEFAULT_ABILITY,
+                    label="Ability",
+                    interactive=True
                 )
-
-    #         data_component = gr.components.Dataframe(
-    #             value=get_baseline_df, 
-    #             headers=COLUMN_NAMES,
-    #             type="pandas", 
-    #             datatype=DATA_TITILE_TYPE,
-    #             interactive=False,
-    #             visible=True,
-    #             height=700,
-    #             )
-    
-    #         choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
-    #         choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component)
-    #         # enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component)
-    #         disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
-    #         checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
-    #         vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
-    #         vbench_validate_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component)
-    #     # Table 1
-    #     with gr.TabItem("Video Quality", elem_id="vbench-tab-table", id=2):
-    #         with gr.Accordion("INSTRUCTION", open=False):
-    #                 citation_button = gr.Textbox(
-    #                     value=QUALITY_CLAIM_TEXT,
-    #                     label="",
-    #                     elem_id="quality-button",
-    #                     lines=2,
-    #                 )
-    #         with gr.Row():
-    #             with gr.Column(scale=1.0):
-    #                 # selection for column part:
-                    
-    #                 checkbox_group_quality = gr.CheckboxGroup(
-    #                     choices=QUALITY_TAB,
-    #                     value=QUALITY_TAB,
-    #                     label="Evaluation Quality Dimension",
-    #                     interactive=True,
-    #                 )
-
-    #         data_component_quality = gr.components.Dataframe(
-    #             value=get_baseline_df_quality, 
-    #             headers=COLUMN_NAMES_QUALITY,
-    #             type="pandas", 
-    #             datatype=DATA_TITILE_TYPE,
-    #             interactive=False,
-    #             visible=True,
-    #             )
-    
-    #         checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality)
-        
-    #     # Table i2v
-    #     with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=3):
-    #         with gr.Accordion("NOTE", open=False):
-    #                 i2v_note_button = gr.Textbox(
-    #                     value=I2V_CLAIM_TEXT,
-    #                     label="",
-    #                     elem_id="quality-button",
-    #                     lines=3,
-    #                 )
-    #         with gr.Row():
-    #             with gr.Column(scale=1.0):
-    #                 # selection for column part:
-    #                 with gr.Row():
-    #                     vbench_team_filter_i2v = gr.Checkbox(
-    #                         label="Sampled by VBench Team (Uncheck to view all submissions)",
-    #                         value=False,
-    #                         interactive=True
-    #                     )
-    #                     vbench_validate_filter_i2v = gr.Checkbox(
-    #                             label="Evaluated by VBench Team (Uncheck to view all submissions)",
-    #                             value=False,
-    #                             interactive=True
-    #                         )
-    #                 checkbox_group_i2v = gr.CheckboxGroup(
-    #                     choices=I2V_TAB,
-    #                     value=I2V_TAB,
-    #                     label="Evaluation Quality Dimension",
-    #                     interactive=True,
-    #                 )
-
-    #         data_component_i2v = gr.components.Dataframe(
-    #             value=get_baseline_df_i2v, 
-    #             headers=COLUMN_NAMES_I2V,
-    #             type="pandas", 
-    #             datatype=I2V_TITILE_TYPE,
-    #             interactive=False,
-    #             visible=True,
-    #             )
-    
-    #         checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
-    #         vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
-    #         vbench_validate_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v)
-        
-    #     with gr.TabItem("📊 VBench-Long", elem_id="vbench-tab-table", id=4):
-    #         with gr.Row():
-    #             with gr.Accordion("INSTRUCTION", open=False):
-    #                 citation_button = gr.Textbox(
-    #                     value=LONG_CLAIM_TEXT,
-    #                     label="",
-    #                     elem_id="long-ins-button",
-    #                     lines=2,
-    #                 )
-    
-    #         gr.Markdown(
-    #             TABLE_INTRODUCTION
-    #         )
-    #         with gr.Row():
-    #             with gr.Column(scale=0.2):
-    #                 choosen_q_long = gr.Button("Select Quality Dimensions")
-    #                 choosen_s_long = gr.Button("Select Semantic Dimensions")
-    #                 enable_b_long = gr.Button("Select All")
-    #                 disable_b_long = gr.Button("Deselect All")
-
-    #             with gr.Column(scale=0.8):
-    #                 with gr.Row():
-    #                     vbench_team_filter_long = gr.Checkbox(
-    #                         label="Sampled by VBench Team (Uncheck to view all submissions)",
-    #                         value=False,
-    #                         interactive=True
-    #                     )
-    #                     vbench_validate_filter_long = gr.Checkbox(
-    #                         label="Evaluated by VBench Team (Uncheck to view all submissions)",
-    #                         value=False,
-    #                         interactive=True
-    #                     )
-    #                 checkbox_group_long = gr.CheckboxGroup(
-    #                     choices=TASK_INFO,
-    #                     value=DEFAULT_INFO,
-    #                     label="Evaluation Dimension",
-    #                     interactive=True,
-    #                 )
-
-    #         data_component = gr.components.Dataframe(
-    #             value=get_baseline_df_long, 
-    #             headers=COLUMN_NAMES,
-    #             type="pandas", 
-    #             datatype=DATA_TITILE_TYPE,
-    #             interactive=False,
-    #             visible=True,
-    #             height=700,
-    #             )
-    
-    #         choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
-    #         choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
-    #         enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
-    #         disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component)
-    #         checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
-    #         vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
-    #         vbench_validate_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component)
-
-    #     # table info
-    #     with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=5):
-    #         gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
-        
-    #     # table submission 
-    #     with gr.TabItem("🚀 [T2V]Submit here! ", elem_id="mvbench-tab-table", id=6):
-    #         gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
-
-    #         with gr.Row():
-    #             gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
-
-    #         with gr.Row():
-    #             gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
-
-    #         with gr.Row():
-    #             gr.Markdown("Here is a required field", elem_classes="markdown-text")
-    #         with gr.Row():
-    #             with gr.Column():
-    #                 model_name_textbox = gr.Textbox(
-    #                     label="Model name", placeholder="Required field"
-    #                     )
-    #                 revision_name_textbox = gr.Textbox(
-    #                     label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
-    #                 )
-    #                 access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
-
-    #             with gr.Column():
-    #                 model_link = gr.Textbox(
-    #                     label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
-    #                 )
-    #                 team_name = gr.Textbox(
-    #                     label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
-    #                 )
-    #                 contact_email = gr.Textbox(
-    #                     label="E-Mail(Will not be displayed)", placeholder="Required field"
-    #                 )
-    #         with gr.Row():
-    #             gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
-    #         with gr.Row():
-    #                 release_time = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
-    #                 model_resolution = gr.Textbox(label="resolution", placeholder="Width x Height")
-    #                 model_fps = gr.Textbox(label="model fps", placeholder="FPS(int)")
-    #                 model_frame = gr.Textbox(label="model frame count", placeholder="INT")
-    #                 model_video_length = gr.Textbox(label="model video length", placeholder="float(2.0)")
-    #                 model_checkpoint = gr.Textbox(label="model checkpoint", placeholder="optional")
-    #                 model_commit_id = gr.Textbox(label="github commit id", placeholder='main')
-    #                 model_video_format = gr.Textbox(label="pipeline format", placeholder='mp4')
-    #         with gr.Column():
-    #             input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
-    #             submit_button = gr.Button("Submit Eval")
-    #             submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
-    #             fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
                 
+            data_component = gr.components.Dataframe(
+                column_widths="auto",  # Automatically adjusts column widths
+                value=get_baseline_df(), 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+            )
     
-    #             submission_result = gr.Markdown()
-    #             # submit_button.click(
-    #             #     add_new_eval,
-    #             #     inputs = [
-    #             #         input_file,
-    #             #         model_name_textbox,
-    #             #         revision_name_textbox,
-    #             #         model_link,
-    #             #         team_name,
-    #             #         contact_email,
-    #             #         release_time,
-    #             #         access_type,
-    #             #         model_resolution,
-    #             #         model_fps,
-    #             #         model_frame,
-    #             #         model_video_length,
-    #             #         model_checkpoint,
-    #             #         model_commit_id,
-    #             #         model_video_format
-    #             #     ],
-    #             #     outputs=[submit_button, submit_succ_button, fail_textbox]
-    #             # )
+            def on_filter_change(model_types, abilities):
+                df = get_baseline_df()
+                # Filter by selected model types
+                df = df[df['Model Type'].isin(model_types)]
+                # Filter by selected abilities 
+                df = df[df['Ability'].isin(abilities)]
+                return gr.Dataframe(
+                    column_widths="auto",  # Automatically adjusts column widths
+                    value=df,
+                    headers=COLUMN_NAMES,
+                    type="pandas",
+                    datatype=DATA_TITILE_TYPE,
+                    interactive=False,
+                    visible=True
+                )
+
+            model_type_filter.change(
+                fn=on_filter_change,
+                inputs=[model_type_filter, ability_filter],
+                outputs=data_component
+            )
+
+            ability_filter.change(
+                fn=on_filter_change,
+                inputs=[model_type_filter, ability_filter], 
+                outputs=data_component
+            )
                 
     #     with gr.TabItem("🚀 [I2V]Submit here! ", elem_id="mvbench-i2v-tab-table", id=7):
     #         gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")

constants.py

@@ -1,193 +1,61 @@
 import os
 # this is .py for store constants 
 MODEL_TYPE = [
-    "video generation", 
-    "3D scene generation", 
-    "4D scene generation"
+    "Video", 
+    "3D", 
+    "4D"
 ]
 DEFAULT_MODEL_TYPE = [
-    "video generation",     
-    "3D scene generation", 
-    "4D scene generation"
+    "Video",     
+    "3D", 
+    "4D"
 ]
 
-MODEL_INFO = [
-    "Model Name (clickable)",
-    "Sampled by",
-    "Evaluated by",
-    "Accessibility",
-    "Date",
-    "Total Score",
-    "Quality Score",
-    "Semantic Score",
-    "Selected Score",
+ABILITY = [
+    "I2V",
+    "T2V"
 ]
-
-MODEL_INFO_TAB_QUALITY = [
-    "Model Name (clickable)",
-    "Quality Score",
-    "Selected Score"
-]
-
-MODEL_INFO_TAB_I2V = [
-    "Model Name (clickable)",
-    "Sampled by",
-    "Evaluated by",
-    "Accessibility",
-    "Date",
-    "Total Score",
-    "I2V Score",
-    "Quality Score",
-    "Selected Score"
+DEFAULT_ABILITY = [
+    "I2V",
+    "T2V"
 ]
 
 TASK_INFO = [
-    "subject consistency",
-    "background consistency",
-    "temporal flickering",
-    "motion smoothness",
-    "dynamic degree",
-    "aesthetic quality",
-    "imaging quality",
-    "object class",
-    "multiple objects",
-    "human action",
-    "color",
-    "spatial relationship",
-    "scene",
-    "appearance style",
-    "temporal style",
-    "overall consistency"
+    "WorldScore-Static",
+    "WorldScore-Dynamic",
+    "Camera Control",
+    "Object Control",
+    "Content Alignment",
+    "3D Consistency",
+    "Photometric Consistency",
+    "Style Consistency",
+    "Subjective Quality",
+    "Motion Accuracy",
+    "Motion Magnitude",
+    "Motion Smoothness",
 ]
-
 DEFAULT_INFO = [
-    "subject consistency",
-    "background consistency",
-    "temporal flickering",
-    "motion smoothness",
-    "dynamic degree",
-    "aesthetic quality",
-    "imaging quality",
-    "object class",
-    "multiple objects",
-    "human action",
-    "color",
-    "spatial relationship",
-    "scene",
-    "appearance style",
-    "temporal style",
-    "overall consistency"
-    ]
-
-QUALITY_LIST = [ 
-    "subject consistency",
-    "background consistency",
-    "temporal flickering",
-    "motion smoothness",
-    "aesthetic quality",
-    "imaging quality",
-    "dynamic degree",]
-
-SEMANTIC_LIST = [
-    "object class",
-    "multiple objects",
-    "human action",
-    "color",
-    "spatial relationship",
-    "scene",
-    "appearance style",
-    "temporal style",
-    "overall consistency"
+    "WorldScore-Static",
 ]
 
-QUALITY_TAB = [ 
-    "subject consistency",
-    "background consistency",
-    "motion smoothness",
-    "aesthetic quality",
-    "imaging quality",
-    "dynamic degree",]
-
-I2V_LIST = [
-    "Video-Text Camera Motion",
-    "Video-Image Subject Consistency",
-    "Video-Image Background Consistency",
-]
-
-I2V_QUALITY_LIST = [
-    "Subject Consistency",
-    "Background Consistency",
-    "Motion Smoothness",
-    "Dynamic Degree",
-    "Aesthetic Quality",
-    "Imaging Quality",
-    # "Temporal Flickering"
-]
-
-I2V_TAB = [
-    "Video-Text Camera Motion",
-    "Video-Image Subject Consistency",
-    "Video-Image Background Consistency",
-    "Subject Consistency",
-    "Background Consistency",
-    "Motion Smoothness",
-    "Dynamic Degree",
-    "Aesthetic Quality",
-    "Imaging Quality",
-    # "Temporal Flickering"
+MODEL_INFO = [
+    "Model Type",
+    "Model Name",
+    "Ability",
+    "Sampled by",
+    "Evaluated by",
+    "Accessibility",
+    "Date",
 ]
 
-DIM_WEIGHT = {
-"subject consistency":1,
-"background consistency":1,
-"temporal flickering":1,
-"motion smoothness":1,
-"aesthetic quality":1,
-"imaging quality":1,
-"dynamic degree":0.5,
-"object class":1,
-"multiple objects":1,
-"human action":1,
-"color":1,
-"spatial relationship":1,
-"scene":1,
-"appearance style":1,
-"temporal style":1,
-"overall consistency":1
-}
-
-DIM_WEIGHT_I2V = {
-"Video-Text Camera Motion": 0.1,
-"Video-Image Subject Consistency": 1,
-"Video-Image Background Consistency": 1,
-"Subject Consistency": 1,
-"Background Consistency": 1,
-"Motion Smoothness": 1,
-"Dynamic Degree": 0.5,
-"Aesthetic Quality": 1,
-"Imaging Quality": 1,
-"Temporal Flickering": 1
-}
-
-SEMANTIC_WEIGHT = 1
-QUALITY_WEIGHT = 4
-I2V_WEIGHT = 1.0
-I2V_QUALITY_WEIGHT = 1.0
-
-DATA_TITILE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
-I2V_TITILE_TYPE =  ['markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
+DATA_TITILE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
 
 SUBMISSION_NAME = "worldscore_leaderboard_submission"
 SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/Howieeeee/", SUBMISSION_NAME)
 CSV_DIR = "./worldscore_leaderboard_submission/results.csv"
-QUALITY_DIR = "./worldscore_leaderboard_submission/quality.csv"
-I2V_DIR = "./worldscore_leaderboard_submission/i2v_results.csv"
-LONG_DIR = "./worldscore_leaderboard_submission/long_debug.csv"
 INFO_DIR = "./worldscore_leaderboard_submission/model_info.csv"
 
 COLUMN_NAMES = MODEL_INFO + TASK_INFO
-COLUMN_NAMES_QUALITY = MODEL_INFO_TAB_QUALITY + QUALITY_TAB
-COLUMN_NAMES_I2V = MODEL_INFO_TAB_I2V + I2V_TAB
 
 LEADERBORAD_INTRODUCTION = """# WorldScore Leaderboard