__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions'] import os import io import gradio as gr import pandas as pd import json import shutil import tempfile import datetime import zipfile import numpy as np from constants import * from huggingface_hub import Repository HF_TOKEN = os.environ.get("HF_TOKEN") global data_component, filter_component def upload_file(files): file_paths = [file.name for file in files] return file_paths # def add_new_eval( # input_file, # model_name_textbox: str, # revision_name_textbox: str, # model_link: str, # team_name: str, # contact_email: str, # access_type: str, # model_publish: str, # model_resolution: str, # model_fps: str, # model_frame: str, # model_video_length: str, # model_checkpoint: str, # model_commit_id: str, # model_video_format: str # ): # if input_file is None: # return "Error! Empty file!" # if model_link == '' or model_name_textbox == '' or contact_email == '': # return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True) # # upload_data=json.loads(input_file) # upload_content = input_file # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") # submission_repo.git_pull() # filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") # now = datetime.datetime.now() # update_time = now.strftime("%Y-%m-%d") # Capture update time # with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f: # f.write(input_file) # # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}")) # csv_data = pd.read_csv(CSV_DIR) # if revision_name_textbox == '': # col = csv_data.shape[0] # model_name = model_name_textbox.replace(',',' ') # else: # model_name = revision_name_textbox.replace(',',' ') # model_name_list = csv_data['Model Name (clickable)'] # name_list = [name.split(']')[0][1:] for name in model_name_list] # if revision_name_textbox not in name_list: # col = csv_data.shape[0] # else: # col = name_list.index(revision_name_textbox) # if model_link == '': # model_name = model_name # no url # else: # model_name = '[' + model_name + '](' + model_link + ')' # os.makedirs(filename, exist_ok=True) # with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref: # zip_ref.extractall(filename) # upload_data = {} # for file in os.listdir(filename): # if file.startswith('.') or file.startswith('__'): # print(f"Skip the file: {file}") # continue # cur_file = os.path.join(filename, file) # if os.path.isdir(cur_file): # for subfile in os.listdir(cur_file): # if subfile.endswith(".json"): # with open(os.path.join(cur_file, subfile)) as ff: # cur_json = json.load(ff) # print(file, type(cur_json)) # if isinstance(cur_json, dict): # print(cur_json.keys()) # for key in cur_json: # upload_data[key.replace('_',' ')] = cur_json[key][0] # print(f"{key}:{cur_json[key][0]}") # elif cur_file.endswith('json'): # with open(cur_file) as ff: # cur_json = json.load(ff) # print(file, type(cur_json)) # if isinstance(cur_json, dict): # print(cur_json.keys()) # for key in cur_json: # upload_data[key.replace('_',' ')] = cur_json[key][0] # print(f"{key}:{cur_json[key][0]}") # # add new data # new_data = [model_name] # print('upload_data:', upload_data) # for key in TASK_INFO: # if key in upload_data: # new_data.append(upload_data[key]) # else: # new_data.append(0) # if team_name =='' or 'vbench' in team_name.lower(): # new_data.append("User Upload") # else: # new_data.append(team_name) # new_data.append(contact_email.replace(',',' and ')) # Add contact email [private] # new_data.append(update_time) # Add the update time # new_data.append(team_name) # new_data.append(access_type) # csv_data.loc[col] = new_data # csv_data = csv_data.to_csv(CSV_DIR, index=False) # with open(INFO_DIR,'a') as f: # f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n") # submission_repo.push_to_hub() # print("success update", model_name) # return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False) # def add_new_eval_i2v( # input_file, # model_name_textbox: str, # revision_name_textbox: str, # model_link: str, # team_name: str, # contact_email: str, # access_type: str, # model_publish: str, # model_resolution: str, # model_fps: str, # model_frame: str, # model_video_length: str, # model_checkpoint: str, # model_commit_id: str, # model_video_format: str # ): # COLNAME2KEY={ # "Video-Text Camera Motion":"camera_motion", # "Video-Image Subject Consistency": "i2v_subject", # "Video-Image Background Consistency": "i2v_background", # "Subject Consistency": "subject_consistency", # "Background Consistency": "background_consistency", # "Motion Smoothness": "motion_smoothness", # "Dynamic Degree": "dynamic_degree", # "Aesthetic Quality": "aesthetic_quality", # "Imaging Quality": "imaging_quality", # "Temporal Flickering": "temporal_flickering" # } # if input_file is None: # return "Error! Empty file!" # if model_link == '' or model_name_textbox == '' or contact_email == '': # return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True) # upload_content = input_file # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") # submission_repo.git_pull() # filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") # now = datetime.datetime.now() # update_time = now.strftime("%Y-%m-%d") # Capture update time # with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f: # f.write(input_file) # # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}")) # csv_data = pd.read_csv(I2V_DIR) # if revision_name_textbox == '': # col = csv_data.shape[0] # model_name = model_name_textbox.replace(',',' ') # else: # model_name = revision_name_textbox.replace(',',' ') # model_name_list = csv_data['Model Name (clickable)'] # name_list = [name.split(']')[0][1:] for name in model_name_list] # if revision_name_textbox not in name_list: # col = csv_data.shape[0] # else: # col = name_list.index(revision_name_textbox) # if model_link == '': # model_name = model_name # no url # else: # model_name = '[' + model_name + '](' + model_link + ')' # os.makedirs(filename, exist_ok=True) # with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref: # zip_ref.extractall(filename) # upload_data = {} # for file in os.listdir(filename): # if file.startswith('.') or file.startswith('__'): # print(f"Skip the file: {file}") # continue # cur_file = os.path.join(filename, file) # if os.path.isdir(cur_file): # for subfile in os.listdir(cur_file): # if subfile.endswith(".json"): # with open(os.path.join(cur_file, subfile)) as ff: # cur_json = json.load(ff) # print(file, type(cur_json)) # if isinstance(cur_json, dict): # print(cur_json.keys()) # for key in cur_json: # upload_data[key] = cur_json[key][0] # print(f"{key}:{cur_json[key][0]}") # elif cur_file.endswith('json'): # with open(cur_file) as ff: # cur_json = json.load(ff) # print(file, type(cur_json)) # if isinstance(cur_json, dict): # print(cur_json.keys()) # for key in cur_json: # upload_data[key] = cur_json[key][0] # print(f"{key}:{cur_json[key][0]}") # # add new data # new_data = [model_name] # print('upload_data:', upload_data) # I2V_HEAD= ["Video-Text Camera Motion", # "Video-Image Subject Consistency", # "Video-Image Background Consistency", # "Subject Consistency", # "Background Consistency", # "Temporal Flickering", # "Motion Smoothness", # "Dynamic Degree", # "Aesthetic Quality", # "Imaging Quality" ] # for key in I2V_HEAD : # sub_key = COLNAME2KEY[key] # if sub_key in upload_data: # new_data.append(upload_data[sub_key]) # else: # new_data.append(0) # if team_name =='' or 'vbench' in team_name.lower(): # new_data.append("User Upload") # else: # new_data.append(team_name) # new_data.append(contact_email.replace(',',' and ')) # Add contact email [private] # new_data.append(update_time) # Add the update time # new_data.append(team_name) # new_data.append(access_type) # csv_data.loc[col] = new_data # csv_data = csv_data.to_csv(I2V_DIR , index=False) # with open(INFO_DIR,'a') as f: # f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n") # submission_repo.push_to_hub() # print("success update", model_name) # return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False) def get_normalized_df(df): # final_score = df.drop('name', axis=1).sum(axis=1) # df.insert(1, 'Overall Score', final_score) normalize_df = df.copy().fillna(0.0) for column in normalize_df.columns[1:-5]: min_val = NORMALIZE_DIC[column]['Min'] max_val = NORMALIZE_DIC[column]['Max'] normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val) return normalize_df def get_normalized_i2v_df(df): normalize_df = df.copy().fillna(0.0) for column in normalize_df.columns[1:-5]: min_val = NORMALIZE_DIC_I2V[column]['Min'] max_val = NORMALIZE_DIC_I2V[column]['Max'] normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val) return normalize_df def calculate_selected_score(df, selected_columns): # selected_score = df[selected_columns].sum(axis=1) selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST] selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST] selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY]) selected_semantic_score = df[selected_SEMANTIC].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_SEMANTIC ]) if selected_quality_score.isna().any().any() and selected_semantic_score.isna().any().any(): selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) return selected_score.fillna(0.0) if selected_quality_score.isna().any().any(): return selected_semantic_score if selected_semantic_score.isna().any().any(): return selected_quality_score # print(selected_semantic_score,selected_quality_score ) selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) return selected_score.fillna(0.0) def calculate_selected_score_i2v(df, selected_columns): # selected_score = df[selected_columns].sum(axis=1) selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST] selected_I2V = [i for i in selected_columns if i in I2V_LIST] selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY]) selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ]) if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any(): selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) return selected_score.fillna(0.0) if selected_quality_score.isna().any().any(): return selected_i2v_score if selected_i2v_score.isna().any().any(): return selected_quality_score # print(selected_i2v_score,selected_quality_score ) selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) return selected_score.fillna(0.0) def get_final_score(df, selected_columns): normalize_df = get_normalized_df(df) #final_score = normalize_df.drop('name', axis=1).sum(axis=1) try: for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1): normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] except: for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1): normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST]) semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ]) final_score = (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) if 'Total Score' in df: df['Total Score'] = final_score else: df.insert(1, 'Total Score', final_score) if 'Semantic Score' in df: df['Semantic Score'] = semantic_score else: df.insert(2, 'Semantic Score', semantic_score) if 'Quality Score' in df: df['Quality Score'] = quality_score else: df.insert(3, 'Quality Score', quality_score) selected_score = calculate_selected_score(normalize_df, selected_columns) if 'Selected Score' in df: df['Selected Score'] = selected_score else: df.insert(1, 'Selected Score', selected_score) return df def get_final_score_i2v(df, selected_columns): normalize_df = get_normalized_i2v_df(df) try: for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1): normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name] except: for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1): normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name] quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST]) i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ]) final_score = (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) if 'Total Score' in df: df['Total Score'] = final_score else: df.insert(1, 'Total Score', final_score) if 'I2V Score' in df: df['I2V Score'] = i2v_score else: df.insert(2, 'I2V Score', i2v_score) if 'Quality Score' in df: df['Quality Score'] = quality_score else: df.insert(3, 'Quality Score', quality_score) selected_score = calculate_selected_score_i2v(normalize_df, selected_columns) if 'Selected Score' in df: df['Selected Score'] = selected_score else: df.insert(1, 'Selected Score', selected_score) # df.loc[df[9:].isnull().any(axis=1), ['Total Score', 'I2V Score']] = 'N.A.' mask = df.iloc[:, 5:-5].isnull().any(axis=1) df.loc[mask, ['Total Score', 'I2V Score','Selected Score' ]] = np.nan # df.fillna('N.A.', inplace=True) return df def get_final_score_quality(df, selected_columns): normalize_df = get_normalized_df(df) for name in normalize_df.drop('Model Name (clickable)', axis=1): normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] quality_score = normalize_df[QUALITY_TAB].sum(axis=1) / sum([DIM_WEIGHT[i] for i in QUALITY_TAB]) if 'Quality Score' in df: df['Quality Score'] = quality_score else: df.insert(1, 'Quality Score', quality_score) # selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns) selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns]) if 'Selected Score' in df: df['Selected Score'] = selected_score else: df.insert(1, 'Selected Score', selected_score) return df def get_baseline_df(): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(CSV_DIR) df = get_final_score(df, checkbox_group.value) df = df.sort_values(by="Selected Score", ascending=False) present_columns = MODEL_INFO + checkbox_group.value # print(present_columns) df = df[present_columns] # Add this line to display the results evaluated by VBench by default df = df[df['Evaluated by'] == 'VBench Team'] df = convert_scores_to_percentage(df) return df def get_baseline_df_quality(): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(QUALITY_DIR) df = get_final_score_quality(df, checkbox_group_quality.value) df = df.sort_values(by="Selected Score", ascending=False) present_columns = MODEL_INFO_TAB_QUALITY + checkbox_group_quality.value df = df[present_columns] df = convert_scores_to_percentage(df) return df def get_baseline_df_i2v(): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(I2V_DIR) df = get_final_score_i2v(df, checkbox_group_i2v.value) df = df.sort_values(by="Selected Score", ascending=False) present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value # df = df[df["Sampled by"] == 'VBench Team'] df = df[present_columns] df = convert_scores_to_percentage(df) return df def get_baseline_df_long(): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(LONG_DIR) df = get_final_score(df, checkbox_group.value) df = df.sort_values(by="Selected Score", ascending=False) present_columns = MODEL_INFO + checkbox_group.value # df = df[df["Sampled by"] == 'VBench Team'] df = df[present_columns] df = convert_scores_to_percentage(df) return df def get_all_df(selected_columns, dir=CSV_DIR): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(dir) df = get_final_score(df, selected_columns) df = df.sort_values(by="Selected Score", ascending=False) return df def get_all_df_quality(selected_columns, dir=QUALITY_DIR): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(dir) df = get_final_score_quality(df, selected_columns) df = df.sort_values(by="Selected Score", ascending=False) return df def get_all_df_i2v(selected_columns, dir=I2V_DIR): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(dir) df = get_final_score_i2v(df, selected_columns) df = df.sort_values(by="Selected Score", ascending=False) return df def get_all_df_long(selected_columns, dir=LONG_DIR): submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") submission_repo.git_pull() df = pd.read_csv(dir) df = get_final_score(df, selected_columns) df = df.sort_values(by="Selected Score", ascending=False) return df def convert_scores_to_percentage(df): # Operate on every column in the DataFrame (except the'name 'column) if "Sampled by" in df.columns: skip_col =3 else: skip_col =1 print(df) for column in df.columns[skip_col:]: # 假设第一列是'name' # if df[column].isdigit(): # print(df[column]) # is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all() valid_numeric_count = pd.to_numeric(df[column], errors='coerce').notna().sum() if valid_numeric_count > 0: df[column] = round(df[column] * 100,2) df[column] = df[column].apply(lambda x: f"{x:05.2f}%" if pd.notna(pd.to_numeric(x, errors='coerce')) else x) # df[column] = df[column].apply(lambda x: f"{x:05.2f}") + '%' return df def choose_all_quailty(): return gr.update(value=QUALITY_LIST) def choose_all_semantic(): return gr.update(value=SEMANTIC_LIST) def disable_all(): return gr.update(value=[]) def enable_all(): return gr.update(value=TASK_INFO) # select function def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbench_team_eval=False): updated_data = get_all_df(selected_columns, CSV_DIR) if vbench_team_sample: updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] if vbench_team_eval: updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team'] #print(updated_data) # columns: selected_columns = [item for item in TASK_INFO if item in selected_columns] present_columns = MODEL_INFO + selected_columns updated_data = updated_data[present_columns] updated_data = updated_data.sort_values(by="Selected Score", ascending=False) updated_data = convert_scores_to_percentage(updated_data) updated_headers = present_columns print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE ) update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] # print(updated_data,present_columns,update_datatype) filter_component = gr.components.Dataframe( value=updated_data, headers=updated_headers, type="pandas", datatype=update_datatype, interactive=False, visible=True, ) return filter_component#.value def on_filter_model_size_method_change_quality(selected_columns): updated_data = get_all_df_quality(selected_columns, QUALITY_DIR) #print(updated_data) # columns: selected_columns = [item for item in QUALITY_TAB if item in selected_columns] present_columns = MODEL_INFO_TAB_QUALITY + selected_columns updated_data = updated_data[present_columns] updated_data = updated_data.sort_values(by="Selected Score", ascending=False) updated_data = convert_scores_to_percentage(updated_data) updated_headers = present_columns update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] # print(updated_data,present_columns,update_datatype) filter_component = gr.components.Dataframe( value=updated_data, headers=updated_headers, type="pandas", datatype=update_datatype, interactive=False, visible=True, ) return filter_component#.value def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample, vbench_team_eval=False): updated_data = get_all_df_i2v(selected_columns, I2V_DIR) if vbench_team_sample: updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] # if vbench_team_eval: # updated_data = updated_data[updated_data['Eval'] == 'VBench Team'] selected_columns = [item for item in I2V_TAB if item in selected_columns] present_columns = MODEL_INFO_TAB_I2V + selected_columns updated_data = updated_data[present_columns] updated_data = updated_data.sort_values(by="Selected Score", ascending=False) updated_data = convert_scores_to_percentage(updated_data) updated_headers = present_columns update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers] # print(updated_data,present_columns,update_datatype) filter_component = gr.components.Dataframe( value=updated_data, headers=updated_headers, type="pandas", datatype=update_datatype, interactive=False, visible=True, ) return filter_component#.value def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample, vbench_team_eval=False): updated_data = get_all_df_long(selected_columns, LONG_DIR) if vbench_team_sample: updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] if vbench_team_eval: updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team'] selected_columns = [item for item in TASK_INFO if item in selected_columns] present_columns = MODEL_INFO + selected_columns updated_data = updated_data[present_columns] updated_data = updated_data.sort_values(by="Selected Score", ascending=False) updated_data = convert_scores_to_percentage(updated_data) updated_headers = present_columns update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] filter_component = gr.components.Dataframe( value=updated_data, headers=updated_headers, type="pandas", datatype=update_datatype, interactive=False, visible=True, ) return filter_component#.value block = gr.Blocks() with block: gr.Markdown( LEADERBORAD_INTRODUCTION ) # with gr.Tabs(elem_classes="tab-buttons") as tabs: # # Table 0 # with gr.TabItem("📊 VBench", elem_id="vbench-tab-table", id=1): # with gr.Row(): # with gr.Accordion("Citation", open=False): # citation_button = gr.Textbox( # value=CITATION_BUTTON_TEXT, # label=CITATION_BUTTON_LABEL, # elem_id="citation-button", # lines=14, # ) # gr.Markdown( # TABLE_INTRODUCTION # ) # with gr.Row(): # with gr.Column(scale=0.2): # choosen_q = gr.Button("Select Quality Dimensions") # choosen_s = gr.Button("Select Semantic Dimensions") # # enable_b = gr.Button("Select All") # disable_b = gr.Button("Deselect All") # with gr.Column(scale=0.8): # vbench_team_filter = gr.Checkbox( # label="Sampled by VBench Team (Uncheck to view all submissions)", # value=False, # interactive=True # ) # vbench_validate_filter = gr.Checkbox( # label="Evaluated by VBench Team (Uncheck to view all submissions)", # value=True, # interactive=True # ) # # selection for column part: # checkbox_group = gr.CheckboxGroup( # choices=TASK_INFO, # value=DEFAULT_INFO, # label="Evaluation Dimension", # interactive=True, # ) # data_component = gr.components.Dataframe( # value=get_baseline_df, # headers=COLUMN_NAMES, # type="pandas", # datatype=DATA_TITILE_TYPE, # interactive=False, # visible=True, # height=700, # ) # choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component) # choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component) # # enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component) # disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) # checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) # vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) # vbench_validate_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) # # Table 1 # with gr.TabItem("Video Quality", elem_id="vbench-tab-table", id=2): # with gr.Accordion("INSTRUCTION", open=False): # citation_button = gr.Textbox( # value=QUALITY_CLAIM_TEXT, # label="", # elem_id="quality-button", # lines=2, # ) # with gr.Row(): # with gr.Column(scale=1.0): # # selection for column part: # checkbox_group_quality = gr.CheckboxGroup( # choices=QUALITY_TAB, # value=QUALITY_TAB, # label="Evaluation Quality Dimension", # interactive=True, # ) # data_component_quality = gr.components.Dataframe( # value=get_baseline_df_quality, # headers=COLUMN_NAMES_QUALITY, # type="pandas", # datatype=DATA_TITILE_TYPE, # interactive=False, # visible=True, # ) # checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality) # # Table i2v # with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=3): # with gr.Accordion("NOTE", open=False): # i2v_note_button = gr.Textbox( # value=I2V_CLAIM_TEXT, # label="", # elem_id="quality-button", # lines=3, # ) # with gr.Row(): # with gr.Column(scale=1.0): # # selection for column part: # with gr.Row(): # vbench_team_filter_i2v = gr.Checkbox( # label="Sampled by VBench Team (Uncheck to view all submissions)", # value=False, # interactive=True # ) # vbench_validate_filter_i2v = gr.Checkbox( # label="Evaluated by VBench Team (Uncheck to view all submissions)", # value=False, # interactive=True # ) # checkbox_group_i2v = gr.CheckboxGroup( # choices=I2V_TAB, # value=I2V_TAB, # label="Evaluation Quality Dimension", # interactive=True, # ) # data_component_i2v = gr.components.Dataframe( # value=get_baseline_df_i2v, # headers=COLUMN_NAMES_I2V, # type="pandas", # datatype=I2V_TITILE_TYPE, # interactive=False, # visible=True, # ) # checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v) # vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v) # vbench_validate_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v) # with gr.TabItem("📊 VBench-Long", elem_id="vbench-tab-table", id=4): # with gr.Row(): # with gr.Accordion("INSTRUCTION", open=False): # citation_button = gr.Textbox( # value=LONG_CLAIM_TEXT, # label="", # elem_id="long-ins-button", # lines=2, # ) # gr.Markdown( # TABLE_INTRODUCTION # ) # with gr.Row(): # with gr.Column(scale=0.2): # choosen_q_long = gr.Button("Select Quality Dimensions") # choosen_s_long = gr.Button("Select Semantic Dimensions") # enable_b_long = gr.Button("Select All") # disable_b_long = gr.Button("Deselect All") # with gr.Column(scale=0.8): # with gr.Row(): # vbench_team_filter_long = gr.Checkbox( # label="Sampled by VBench Team (Uncheck to view all submissions)", # value=False, # interactive=True # ) # vbench_validate_filter_long = gr.Checkbox( # label="Evaluated by VBench Team (Uncheck to view all submissions)", # value=False, # interactive=True # ) # checkbox_group_long = gr.CheckboxGroup( # choices=TASK_INFO, # value=DEFAULT_INFO, # label="Evaluation Dimension", # interactive=True, # ) # data_component = gr.components.Dataframe( # value=get_baseline_df_long, # headers=COLUMN_NAMES, # type="pandas", # datatype=DATA_TITILE_TYPE, # interactive=False, # visible=True, # height=700, # ) # choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) # choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) # enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) # disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) # checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component) # vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component) # vbench_validate_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component) # # table info # with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=5): # gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text") # # table submission # with gr.TabItem("🚀 [T2V]Submit here! ", elem_id="mvbench-tab-table", id=6): # gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text") # with gr.Row(): # gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") # with gr.Row(): # gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text") # with gr.Row(): # gr.Markdown("Here is a required field", elem_classes="markdown-text") # with gr.Row(): # with gr.Column(): # model_name_textbox = gr.Textbox( # label="Model name", placeholder="Required field" # ) # revision_name_textbox = gr.Textbox( # label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line" # ) # access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.") # with gr.Column(): # model_link = gr.Textbox( # label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed." # ) # team_name = gr.Textbox( # label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload" # ) # contact_email = gr.Textbox( # label="E-Mail(Will not be displayed)", placeholder="Required field" # ) # with gr.Row(): # gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text") # with gr.Row(): # release_time = gr.Textbox(label="Time of Publish", placeholder="1970-01-01") # model_resolution = gr.Textbox(label="resolution", placeholder="Width x Height") # model_fps = gr.Textbox(label="model fps", placeholder="FPS(int)") # model_frame = gr.Textbox(label="model frame count", placeholder="INT") # model_video_length = gr.Textbox(label="model video length", placeholder="float(2.0)") # model_checkpoint = gr.Textbox(label="model checkpoint", placeholder="optional") # model_commit_id = gr.Textbox(label="github commit id", placeholder='main') # model_video_format = gr.Textbox(label="pipeline format", placeholder='mp4') # with gr.Column(): # input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary') # submit_button = gr.Button("Submit Eval") # submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False) # fail_textbox = gr.Markdown('Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.', elem_classes="markdown-text",visible=False) # submission_result = gr.Markdown() # # submit_button.click( # # add_new_eval, # # inputs = [ # # input_file, # # model_name_textbox, # # revision_name_textbox, # # model_link, # # team_name, # # contact_email, # # release_time, # # access_type, # # model_resolution, # # model_fps, # # model_frame, # # model_video_length, # # model_checkpoint, # # model_commit_id, # # model_video_format # # ], # # outputs=[submit_button, submit_succ_button, fail_textbox] # # ) # with gr.TabItem("🚀 [I2V]Submit here! ", elem_id="mvbench-i2v-tab-table", id=7): # gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text") # with gr.Row(): # gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") # with gr.Row(): # gr.Markdown("# ✉️✨ Submit your i2v model evaluation json file here!", elem_classes="markdown-text") # with gr.Row(): # gr.Markdown("Here is a required field", elem_classes="markdown-text") # with gr.Row(): # with gr.Column(): # model_name_textbox_i2v = gr.Textbox( # label="Model name", placeholder="Required field" # ) # revision_name_textbox_i2v = gr.Textbox( # label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line" # ) # access_type_i2v = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.") # with gr.Column(): # model_link_i2v = gr.Textbox( # label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed." # ) # team_name_i2v = gr.Textbox( # label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload" # ) # contact_email_i2v = gr.Textbox( # label="E-Mail(Will not be displayed)", placeholder="Required field" # ) # with gr.Row(): # gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text") # with gr.Row(): # release_time_i2v = gr.Textbox(label="Time of Publish", placeholder="1970-01-01") # model_resolution_i2v = gr.Textbox(label="resolution", placeholder="Width x Height") # model_fps_i2v = gr.Textbox(label="model fps", placeholder="FPS(int)") # model_frame_i2v = gr.Textbox(label="model frame count", placeholder="INT") # model_video_length_i2v = gr.Textbox(label="model video length", placeholder="float(2.0)") # model_checkpoint_i2v = gr.Textbox(label="model checkpoint", placeholder="optional") # model_commit_id_i2v = gr.Textbox(label="github commit id", placeholder='main') # model_video_format_i2v = gr.Textbox(label="pipeline format", placeholder='mp4') # with gr.Column(): # input_file_i2v = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary') # submit_button_i2v = gr.Button("Submit Eval") # submit_succ_button_i2v = gr.Markdown("Submit Success! Please press refresh and retfurn to LeaderBoard!", visible=False) # fail_textbox_i2v = gr.Markdown('Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.', elem_classes="markdown-text",visible=False) # submission_result_i2v = gr.Markdown() # # submit_button_i2v.click( # # add_new_eval_i2v, # # inputs = [ # # input_file_i2v, # # model_name_textbox_i2v, # # revision_name_textbox_i2v, # # model_link_i2v, # # team_name_i2v, # # contact_email_i2v, # # release_time_i2v, # # access_type_i2v, # # model_resolution_i2v, # # model_fps_i2v, # # model_frame_i2v, # # model_video_length_i2v, # # model_checkpoint_i2v, # # model_commit_id_i2v, # # model_video_format_i2v # # ], # # outputs=[submit_button_i2v, submit_succ_button_i2v, fail_textbox_i2v] # # ) # def refresh_data(): # value1 = get_baseline_df() # return value1 # with gr.Row(): # data_run = gr.Button("Refresh") # data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component) block.launch()