| __all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions'] | |
| import os | |
| import io | |
| import gradio as gr | |
| import pandas as pd | |
| import json | |
| import shutil | |
| import tempfile | |
| import datetime | |
| import zipfile | |
| import numpy as np | |
| from constants import * | |
| from huggingface_hub import Repository | |
| HF_TOKEN = os.environ.get("HF_TOKEN") | |
| global data_component, filter_component | |
| def upload_file(files): | |
| file_paths = [file.name for file in files] | |
| return file_paths | |
| # def add_new_eval( | |
| # input_file, | |
| # model_name_textbox: str, | |
| # revision_name_textbox: str, | |
| # model_link: str, | |
| # team_name: str, | |
| # contact_email: str, | |
| # access_type: str, | |
| # model_publish: str, | |
| # model_resolution: str, | |
| # model_fps: str, | |
| # model_frame: str, | |
| # model_video_length: str, | |
| # model_checkpoint: str, | |
| # model_commit_id: str, | |
| # model_video_format: str | |
| # ): | |
| # if input_file is None: | |
| # return "Error! Empty file!" | |
| # if model_link == '' or model_name_textbox == '' or contact_email == '': | |
| # return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True) | |
| # # upload_data=json.loads(input_file) | |
| # upload_content = input_file | |
| # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| # submission_repo.git_pull() | |
| # filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") | |
| # now = datetime.datetime.now() | |
| # update_time = now.strftime("%Y-%m-%d") # Capture update time | |
| # with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f: | |
| # f.write(input_file) | |
| # # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}")) | |
| # csv_data = pd.read_csv(CSV_DIR) | |
| # if revision_name_textbox == '': | |
| # col = csv_data.shape[0] | |
| # model_name = model_name_textbox.replace(',',' ') | |
| # else: | |
| # model_name = revision_name_textbox.replace(',',' ') | |
| # model_name_list = csv_data['Model Name (clickable)'] | |
| # name_list = [name.split(']')[0][1:] for name in model_name_list] | |
| # if revision_name_textbox not in name_list: | |
| # col = csv_data.shape[0] | |
| # else: | |
| # col = name_list.index(revision_name_textbox) | |
| # if model_link == '': | |
| # model_name = model_name # no url | |
| # else: | |
| # model_name = '[' + model_name + '](' + model_link + ')' | |
| # os.makedirs(filename, exist_ok=True) | |
| # with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref: | |
| # zip_ref.extractall(filename) | |
| # upload_data = {} | |
| # for file in os.listdir(filename): | |
| # if file.startswith('.') or file.startswith('__'): | |
| # print(f"Skip the file: {file}") | |
| # continue | |
| # cur_file = os.path.join(filename, file) | |
| # if os.path.isdir(cur_file): | |
| # for subfile in os.listdir(cur_file): | |
| # if subfile.endswith(".json"): | |
| # with open(os.path.join(cur_file, subfile)) as ff: | |
| # cur_json = json.load(ff) | |
| # print(file, type(cur_json)) | |
| # if isinstance(cur_json, dict): | |
| # print(cur_json.keys()) | |
| # for key in cur_json: | |
| # upload_data[key.replace('_',' ')] = cur_json[key][0] | |
| # print(f"{key}:{cur_json[key][0]}") | |
| # elif cur_file.endswith('json'): | |
| # with open(cur_file) as ff: | |
| # cur_json = json.load(ff) | |
| # print(file, type(cur_json)) | |
| # if isinstance(cur_json, dict): | |
| # print(cur_json.keys()) | |
| # for key in cur_json: | |
| # upload_data[key.replace('_',' ')] = cur_json[key][0] | |
| # print(f"{key}:{cur_json[key][0]}") | |
| # # add new data | |
| # new_data = [model_name] | |
| # print('upload_data:', upload_data) | |
| # for key in TASK_INFO: | |
| # if key in upload_data: | |
| # new_data.append(upload_data[key]) | |
| # else: | |
| # new_data.append(0) | |
| # if team_name =='' or 'vbench' in team_name.lower(): | |
| # new_data.append("User Upload") | |
| # else: | |
| # new_data.append(team_name) | |
| # new_data.append(contact_email.replace(',',' and ')) # Add contact email [private] | |
| # new_data.append(update_time) # Add the update time | |
| # new_data.append(team_name) | |
| # new_data.append(access_type) | |
| # csv_data.loc[col] = new_data | |
| # csv_data = csv_data.to_csv(CSV_DIR, index=False) | |
| # with open(INFO_DIR,'a') as f: | |
| # f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n") | |
| # submission_repo.push_to_hub() | |
| # print("success update", model_name) | |
| # return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False) | |
| # def add_new_eval_i2v( | |
| # input_file, | |
| # model_name_textbox: str, | |
| # revision_name_textbox: str, | |
| # model_link: str, | |
| # team_name: str, | |
| # contact_email: str, | |
| # access_type: str, | |
| # model_publish: str, | |
| # model_resolution: str, | |
| # model_fps: str, | |
| # model_frame: str, | |
| # model_video_length: str, | |
| # model_checkpoint: str, | |
| # model_commit_id: str, | |
| # model_video_format: str | |
| # ): | |
| # COLNAME2KEY={ | |
| # "Video-Text Camera Motion":"camera_motion", | |
| # "Video-Image Subject Consistency": "i2v_subject", | |
| # "Video-Image Background Consistency": "i2v_background", | |
| # "Subject Consistency": "subject_consistency", | |
| # "Background Consistency": "background_consistency", | |
| # "Motion Smoothness": "motion_smoothness", | |
| # "Dynamic Degree": "dynamic_degree", | |
| # "Aesthetic Quality": "aesthetic_quality", | |
| # "Imaging Quality": "imaging_quality", | |
| # "Temporal Flickering": "temporal_flickering" | |
| # } | |
| # if input_file is None: | |
| # return "Error! Empty file!" | |
| # if model_link == '' or model_name_textbox == '' or contact_email == '': | |
| # return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True) | |
| # upload_content = input_file | |
| # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| # submission_repo.git_pull() | |
| # filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") | |
| # now = datetime.datetime.now() | |
| # update_time = now.strftime("%Y-%m-%d") # Capture update time | |
| # with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f: | |
| # f.write(input_file) | |
| # # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}")) | |
| # csv_data = pd.read_csv(I2V_DIR) | |
| # if revision_name_textbox == '': | |
| # col = csv_data.shape[0] | |
| # model_name = model_name_textbox.replace(',',' ') | |
| # else: | |
| # model_name = revision_name_textbox.replace(',',' ') | |
| # model_name_list = csv_data['Model Name (clickable)'] | |
| # name_list = [name.split(']')[0][1:] for name in model_name_list] | |
| # if revision_name_textbox not in name_list: | |
| # col = csv_data.shape[0] | |
| # else: | |
| # col = name_list.index(revision_name_textbox) | |
| # if model_link == '': | |
| # model_name = model_name # no url | |
| # else: | |
| # model_name = '[' + model_name + '](' + model_link + ')' | |
| # os.makedirs(filename, exist_ok=True) | |
| # with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref: | |
| # zip_ref.extractall(filename) | |
| # upload_data = {} | |
| # for file in os.listdir(filename): | |
| # if file.startswith('.') or file.startswith('__'): | |
| # print(f"Skip the file: {file}") | |
| # continue | |
| # cur_file = os.path.join(filename, file) | |
| # if os.path.isdir(cur_file): | |
| # for subfile in os.listdir(cur_file): | |
| # if subfile.endswith(".json"): | |
| # with open(os.path.join(cur_file, subfile)) as ff: | |
| # cur_json = json.load(ff) | |
| # print(file, type(cur_json)) | |
| # if isinstance(cur_json, dict): | |
| # print(cur_json.keys()) | |
| # for key in cur_json: | |
| # upload_data[key] = cur_json[key][0] | |
| # print(f"{key}:{cur_json[key][0]}") | |
| # elif cur_file.endswith('json'): | |
| # with open(cur_file) as ff: | |
| # cur_json = json.load(ff) | |
| # print(file, type(cur_json)) | |
| # if isinstance(cur_json, dict): | |
| # print(cur_json.keys()) | |
| # for key in cur_json: | |
| # upload_data[key] = cur_json[key][0] | |
| # print(f"{key}:{cur_json[key][0]}") | |
| # # add new data | |
| # new_data = [model_name] | |
| # print('upload_data:', upload_data) | |
| # I2V_HEAD= ["Video-Text Camera Motion", | |
| # "Video-Image Subject Consistency", | |
| # "Video-Image Background Consistency", | |
| # "Subject Consistency", | |
| # "Background Consistency", | |
| # "Temporal Flickering", | |
| # "Motion Smoothness", | |
| # "Dynamic Degree", | |
| # "Aesthetic Quality", | |
| # "Imaging Quality" ] | |
| # for key in I2V_HEAD : | |
| # sub_key = COLNAME2KEY[key] | |
| # if sub_key in upload_data: | |
| # new_data.append(upload_data[sub_key]) | |
| # else: | |
| # new_data.append(0) | |
| # if team_name =='' or 'vbench' in team_name.lower(): | |
| # new_data.append("User Upload") | |
| # else: | |
| # new_data.append(team_name) | |
| # new_data.append(contact_email.replace(',',' and ')) # Add contact email [private] | |
| # new_data.append(update_time) # Add the update time | |
| # new_data.append(team_name) | |
| # new_data.append(access_type) | |
| # csv_data.loc[col] = new_data | |
| # csv_data = csv_data.to_csv(I2V_DIR , index=False) | |
| # with open(INFO_DIR,'a') as f: | |
| # f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n") | |
| # submission_repo.push_to_hub() | |
| # print("success update", model_name) | |
| # return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False) | |
| def get_normalized_df(df): | |
| # final_score = df.drop('name', axis=1).sum(axis=1) | |
| # df.insert(1, 'Overall Score', final_score) | |
| normalize_df = df.copy().fillna(0.0) | |
| for column in normalize_df.columns[1:-5]: | |
| min_val = NORMALIZE_DIC[column]['Min'] | |
| max_val = NORMALIZE_DIC[column]['Max'] | |
| normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val) | |
| return normalize_df | |
| def get_normalized_i2v_df(df): | |
| normalize_df = df.copy().fillna(0.0) | |
| for column in normalize_df.columns[1:-5]: | |
| min_val = NORMALIZE_DIC_I2V[column]['Min'] | |
| max_val = NORMALIZE_DIC_I2V[column]['Max'] | |
| normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val) | |
| return normalize_df | |
| def calculate_selected_score(df, selected_columns): | |
| # selected_score = df[selected_columns].sum(axis=1) | |
| selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST] | |
| selected_SEMANTIC = [i for i in selected_columns if i in SEMANTIC_LIST] | |
| selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_QUALITY]) | |
| selected_semantic_score = df[selected_SEMANTIC].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_SEMANTIC ]) | |
| if selected_quality_score.isna().any().any() and selected_semantic_score.isna().any().any(): | |
| selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| if selected_quality_score.isna().any().any(): | |
| return selected_semantic_score | |
| if selected_semantic_score.isna().any().any(): | |
| return selected_quality_score | |
| # print(selected_semantic_score,selected_quality_score ) | |
| selected_score = (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| def calculate_selected_score_i2v(df, selected_columns): | |
| # selected_score = df[selected_columns].sum(axis=1) | |
| selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST] | |
| selected_I2V = [i for i in selected_columns if i in I2V_LIST] | |
| selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY]) | |
| selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ]) | |
| if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any(): | |
| selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| if selected_quality_score.isna().any().any(): | |
| return selected_i2v_score | |
| if selected_i2v_score.isna().any().any(): | |
| return selected_quality_score | |
| # print(selected_i2v_score,selected_quality_score ) | |
| selected_score = (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) | |
| return selected_score.fillna(0.0) | |
| def get_final_score(df, selected_columns): | |
| normalize_df = get_normalized_df(df) | |
| #final_score = normalize_df.drop('name', axis=1).sum(axis=1) | |
| try: | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] | |
| except: | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] | |
| quality_score = normalize_df[QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in QUALITY_LIST]) | |
| semantic_score = normalize_df[SEMANTIC_LIST].sum(axis=1)/sum([DIM_WEIGHT[i] for i in SEMANTIC_LIST ]) | |
| final_score = (quality_score * QUALITY_WEIGHT + semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT) | |
| if 'Total Score' in df: | |
| df['Total Score'] = final_score | |
| else: | |
| df.insert(1, 'Total Score', final_score) | |
| if 'Semantic Score' in df: | |
| df['Semantic Score'] = semantic_score | |
| else: | |
| df.insert(2, 'Semantic Score', semantic_score) | |
| if 'Quality Score' in df: | |
| df['Quality Score'] = quality_score | |
| else: | |
| df.insert(3, 'Quality Score', quality_score) | |
| selected_score = calculate_selected_score(normalize_df, selected_columns) | |
| if 'Selected Score' in df: | |
| df['Selected Score'] = selected_score | |
| else: | |
| df.insert(1, 'Selected Score', selected_score) | |
| return df | |
| def get_final_score_i2v(df, selected_columns): | |
| normalize_df = get_normalized_i2v_df(df) | |
| try: | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1).drop("Evaluated by", axis=1).drop("Accessibility", axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name] | |
| except: | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1).drop("Sampled by", axis=1).drop('Mail', axis=1).drop('Date',axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name] | |
| quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST]) | |
| i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ]) | |
| final_score = (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT) | |
| if 'Total Score' in df: | |
| df['Total Score'] = final_score | |
| else: | |
| df.insert(1, 'Total Score', final_score) | |
| if 'I2V Score' in df: | |
| df['I2V Score'] = i2v_score | |
| else: | |
| df.insert(2, 'I2V Score', i2v_score) | |
| if 'Quality Score' in df: | |
| df['Quality Score'] = quality_score | |
| else: | |
| df.insert(3, 'Quality Score', quality_score) | |
| selected_score = calculate_selected_score_i2v(normalize_df, selected_columns) | |
| if 'Selected Score' in df: | |
| df['Selected Score'] = selected_score | |
| else: | |
| df.insert(1, 'Selected Score', selected_score) | |
| # df.loc[df[9:].isnull().any(axis=1), ['Total Score', 'I2V Score']] = 'N.A.' | |
| mask = df.iloc[:, 5:-5].isnull().any(axis=1) | |
| df.loc[mask, ['Total Score', 'I2V Score','Selected Score' ]] = np.nan | |
| # df.fillna('N.A.', inplace=True) | |
| return df | |
| def get_final_score_quality(df, selected_columns): | |
| normalize_df = get_normalized_df(df) | |
| for name in normalize_df.drop('Model Name (clickable)', axis=1): | |
| normalize_df[name] = normalize_df[name]*DIM_WEIGHT[name] | |
| quality_score = normalize_df[QUALITY_TAB].sum(axis=1) / sum([DIM_WEIGHT[i] for i in QUALITY_TAB]) | |
| if 'Quality Score' in df: | |
| df['Quality Score'] = quality_score | |
| else: | |
| df.insert(1, 'Quality Score', quality_score) | |
| # selected_score = normalize_df[selected_columns].sum(axis=1) / len(selected_columns) | |
| selected_score = normalize_df[selected_columns].sum(axis=1)/sum([DIM_WEIGHT[i] for i in selected_columns]) | |
| if 'Selected Score' in df: | |
| df['Selected Score'] = selected_score | |
| else: | |
| df.insert(1, 'Selected Score', selected_score) | |
| return df | |
| def get_baseline_df(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(CSV_DIR) | |
| df = get_final_score(df, checkbox_group.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO + checkbox_group.value | |
| # print(present_columns) | |
| df = df[present_columns] | |
| # Add this line to display the results evaluated by VBench by default | |
| df = df[df['Evaluated by'] == 'VBench Team'] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_baseline_df_quality(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(QUALITY_DIR) | |
| df = get_final_score_quality(df, checkbox_group_quality.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO_TAB_QUALITY + checkbox_group_quality.value | |
| df = df[present_columns] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_baseline_df_i2v(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(I2V_DIR) | |
| df = get_final_score_i2v(df, checkbox_group_i2v.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value | |
| # df = df[df["Sampled by"] == 'VBench Team'] | |
| df = df[present_columns] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_baseline_df_long(): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(LONG_DIR) | |
| df = get_final_score(df, checkbox_group.value) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| present_columns = MODEL_INFO + checkbox_group.value | |
| # df = df[df["Sampled by"] == 'VBench Team'] | |
| df = df[present_columns] | |
| df = convert_scores_to_percentage(df) | |
| return df | |
| def get_all_df(selected_columns, dir=CSV_DIR): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def get_all_df_quality(selected_columns, dir=QUALITY_DIR): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score_quality(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def get_all_df_i2v(selected_columns, dir=I2V_DIR): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score_i2v(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def get_all_df_long(selected_columns, dir=LONG_DIR): | |
| submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset") | |
| submission_repo.git_pull() | |
| df = pd.read_csv(dir) | |
| df = get_final_score(df, selected_columns) | |
| df = df.sort_values(by="Selected Score", ascending=False) | |
| return df | |
| def convert_scores_to_percentage(df): | |
| # Operate on every column in the DataFrame (except the'name 'column) | |
| if "Sampled by" in df.columns: | |
| skip_col =3 | |
| else: | |
| skip_col =1 | |
| print(df) | |
| for column in df.columns[skip_col:]: # 假设第一列是'name' | |
| # if df[column].isdigit(): | |
| # print(df[column]) | |
| # is_numeric = pd.to_numeric(df[column], errors='coerce').notna().all() | |
| valid_numeric_count = pd.to_numeric(df[column], errors='coerce').notna().sum() | |
| if valid_numeric_count > 0: | |
| df[column] = round(df[column] * 100,2) | |
| df[column] = df[column].apply(lambda x: f"{x:05.2f}%" if pd.notna(pd.to_numeric(x, errors='coerce')) else x) | |
| # df[column] = df[column].apply(lambda x: f"{x:05.2f}") + '%' | |
| return df | |
| def choose_all_quailty(): | |
| return gr.update(value=QUALITY_LIST) | |
| def choose_all_semantic(): | |
| return gr.update(value=SEMANTIC_LIST) | |
| def disable_all(): | |
| return gr.update(value=[]) | |
| def enable_all(): | |
| return gr.update(value=TASK_INFO) | |
| # select function | |
| def on_filter_model_size_method_change(selected_columns, vbench_team_sample, vbench_team_eval=False): | |
| updated_data = get_all_df(selected_columns, CSV_DIR) | |
| if vbench_team_sample: | |
| updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] | |
| if vbench_team_eval: | |
| updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team'] | |
| #print(updated_data) | |
| # columns: | |
| selected_columns = [item for item in TASK_INFO if item in selected_columns] | |
| present_columns = MODEL_INFO + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| print(COLUMN_NAMES,updated_headers,DATA_TITILE_TYPE ) | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] | |
| # print(updated_data,present_columns,update_datatype) | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| def on_filter_model_size_method_change_quality(selected_columns): | |
| updated_data = get_all_df_quality(selected_columns, QUALITY_DIR) | |
| #print(updated_data) | |
| # columns: | |
| selected_columns = [item for item in QUALITY_TAB if item in selected_columns] | |
| present_columns = MODEL_INFO_TAB_QUALITY + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] | |
| # print(updated_data,present_columns,update_datatype) | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| def on_filter_model_size_method_change_i2v(selected_columns,vbench_team_sample, vbench_team_eval=False): | |
| updated_data = get_all_df_i2v(selected_columns, I2V_DIR) | |
| if vbench_team_sample: | |
| updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] | |
| # if vbench_team_eval: | |
| # updated_data = updated_data[updated_data['Eval'] == 'VBench Team'] | |
| selected_columns = [item for item in I2V_TAB if item in selected_columns] | |
| present_columns = MODEL_INFO_TAB_I2V + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES_I2V.index(x)] for x in updated_headers] | |
| # print(updated_data,present_columns,update_datatype) | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| def on_filter_model_size_method_change_long(selected_columns, vbench_team_sample, vbench_team_eval=False): | |
| updated_data = get_all_df_long(selected_columns, LONG_DIR) | |
| if vbench_team_sample: | |
| updated_data = updated_data[updated_data["Sampled by"] == 'VBench Team'] | |
| if vbench_team_eval: | |
| updated_data = updated_data[updated_data['Evaluated by'] == 'VBench Team'] | |
| selected_columns = [item for item in TASK_INFO if item in selected_columns] | |
| present_columns = MODEL_INFO + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by="Selected Score", ascending=False) | |
| updated_data = convert_scores_to_percentage(updated_data) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component#.value | |
| block = gr.Blocks() | |
| with block: | |
| gr.Markdown( | |
| LEADERBORAD_INTRODUCTION | |
| ) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| # Table 0 | |
| with gr.TabItem("🏅 WorldScore Benchmark", elem_id="worldscore-tab-table", id=1): | |
| with gr.Column(): | |
| model_type_filter = gr.CheckboxGroup( | |
| choices=MODEL_TYPE, | |
| value=DEFAULT_MODEL_TYPE, | |
| label="Model Type", | |
| interactive=True | |
| ) | |
| # selection for column part: | |
| checkbox_group = gr.CheckboxGroup( | |
| choices=TASK_INFO, | |
| value=DEFAULT_INFO, | |
| label="Evaluation Dimension", | |
| interactive=True, | |
| ) | |
| # data_component = gr.components.Dataframe( | |
| # value=get_baseline_df, | |
| # headers=COLUMN_NAMES, | |
| # type="pandas", | |
| # datatype=DATA_TITILE_TYPE, | |
| # interactive=False, | |
| # visible=True, | |
| # height=700, | |
| # ) | |
| # choosen_q.click(choose_all_quailty, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component) | |
| # choosen_s.click(choose_all_semantic, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter,vbench_validate_filter], outputs=data_component) | |
| # # enable_b.click(enable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter], outputs=data_component) | |
| # disable_b.click(disable_all, inputs=None, outputs=[checkbox_group]).then(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) | |
| # checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[ checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) | |
| # vbench_team_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) | |
| # vbench_validate_filter.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group, vbench_team_filter, vbench_validate_filter], outputs=data_component) | |
| # # Table 1 | |
| # with gr.TabItem("Video Quality", elem_id="vbench-tab-table", id=2): | |
| # with gr.Accordion("INSTRUCTION", open=False): | |
| # citation_button = gr.Textbox( | |
| # value=QUALITY_CLAIM_TEXT, | |
| # label="", | |
| # elem_id="quality-button", | |
| # lines=2, | |
| # ) | |
| # with gr.Row(): | |
| # with gr.Column(scale=1.0): | |
| # # selection for column part: | |
| # checkbox_group_quality = gr.CheckboxGroup( | |
| # choices=QUALITY_TAB, | |
| # value=QUALITY_TAB, | |
| # label="Evaluation Quality Dimension", | |
| # interactive=True, | |
| # ) | |
| # data_component_quality = gr.components.Dataframe( | |
| # value=get_baseline_df_quality, | |
| # headers=COLUMN_NAMES_QUALITY, | |
| # type="pandas", | |
| # datatype=DATA_TITILE_TYPE, | |
| # interactive=False, | |
| # visible=True, | |
| # ) | |
| # checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality) | |
| # # Table i2v | |
| # with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=3): | |
| # with gr.Accordion("NOTE", open=False): | |
| # i2v_note_button = gr.Textbox( | |
| # value=I2V_CLAIM_TEXT, | |
| # label="", | |
| # elem_id="quality-button", | |
| # lines=3, | |
| # ) | |
| # with gr.Row(): | |
| # with gr.Column(scale=1.0): | |
| # # selection for column part: | |
| # with gr.Row(): | |
| # vbench_team_filter_i2v = gr.Checkbox( | |
| # label="Sampled by VBench Team (Uncheck to view all submissions)", | |
| # value=False, | |
| # interactive=True | |
| # ) | |
| # vbench_validate_filter_i2v = gr.Checkbox( | |
| # label="Evaluated by VBench Team (Uncheck to view all submissions)", | |
| # value=False, | |
| # interactive=True | |
| # ) | |
| # checkbox_group_i2v = gr.CheckboxGroup( | |
| # choices=I2V_TAB, | |
| # value=I2V_TAB, | |
| # label="Evaluation Quality Dimension", | |
| # interactive=True, | |
| # ) | |
| # data_component_i2v = gr.components.Dataframe( | |
| # value=get_baseline_df_i2v, | |
| # headers=COLUMN_NAMES_I2V, | |
| # type="pandas", | |
| # datatype=I2V_TITILE_TYPE, | |
| # interactive=False, | |
| # visible=True, | |
| # ) | |
| # checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v) | |
| # vbench_team_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v) | |
| # vbench_validate_filter_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v, vbench_team_filter_i2v,vbench_validate_filter_i2v], outputs=data_component_i2v) | |
| # with gr.TabItem("📊 VBench-Long", elem_id="vbench-tab-table", id=4): | |
| # with gr.Row(): | |
| # with gr.Accordion("INSTRUCTION", open=False): | |
| # citation_button = gr.Textbox( | |
| # value=LONG_CLAIM_TEXT, | |
| # label="", | |
| # elem_id="long-ins-button", | |
| # lines=2, | |
| # ) | |
| # gr.Markdown( | |
| # TABLE_INTRODUCTION | |
| # ) | |
| # with gr.Row(): | |
| # with gr.Column(scale=0.2): | |
| # choosen_q_long = gr.Button("Select Quality Dimensions") | |
| # choosen_s_long = gr.Button("Select Semantic Dimensions") | |
| # enable_b_long = gr.Button("Select All") | |
| # disable_b_long = gr.Button("Deselect All") | |
| # with gr.Column(scale=0.8): | |
| # with gr.Row(): | |
| # vbench_team_filter_long = gr.Checkbox( | |
| # label="Sampled by VBench Team (Uncheck to view all submissions)", | |
| # value=False, | |
| # interactive=True | |
| # ) | |
| # vbench_validate_filter_long = gr.Checkbox( | |
| # label="Evaluated by VBench Team (Uncheck to view all submissions)", | |
| # value=False, | |
| # interactive=True | |
| # ) | |
| # checkbox_group_long = gr.CheckboxGroup( | |
| # choices=TASK_INFO, | |
| # value=DEFAULT_INFO, | |
| # label="Evaluation Dimension", | |
| # interactive=True, | |
| # ) | |
| # data_component = gr.components.Dataframe( | |
| # value=get_baseline_df_long, | |
| # headers=COLUMN_NAMES, | |
| # type="pandas", | |
| # datatype=DATA_TITILE_TYPE, | |
| # interactive=False, | |
| # visible=True, | |
| # height=700, | |
| # ) | |
| # choosen_q_long.click(choose_all_quailty, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) | |
| # choosen_s_long.click(choose_all_semantic, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) | |
| # enable_b_long.click(enable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) | |
| # disable_b_long.click(disable_all, inputs=None, outputs=[checkbox_group_long]).then(fn=on_filter_model_size_method_change_long, inputs=[ checkbox_group_long, vbench_team_filter_long, vbench_validate_filter_long], outputs=data_component) | |
| # checkbox_group_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component) | |
| # vbench_team_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component) | |
| # vbench_validate_filter_long.change(fn=on_filter_model_size_method_change_long, inputs=[checkbox_group_long, vbench_team_filter_long,vbench_validate_filter_long], outputs=data_component) | |
| # # table info | |
| # with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=5): | |
| # gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text") | |
| # # table submission | |
| # with gr.TabItem("🚀 [T2V]Submit here! ", elem_id="mvbench-tab-table", id=6): | |
| # gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # gr.Markdown("Here is a required field", elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # with gr.Column(): | |
| # model_name_textbox = gr.Textbox( | |
| # label="Model name", placeholder="Required field" | |
| # ) | |
| # revision_name_textbox = gr.Textbox( | |
| # label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line" | |
| # ) | |
| # access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.") | |
| # with gr.Column(): | |
| # model_link = gr.Textbox( | |
| # label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed." | |
| # ) | |
| # team_name = gr.Textbox( | |
| # label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload" | |
| # ) | |
| # contact_email = gr.Textbox( | |
| # label="E-Mail(Will not be displayed)", placeholder="Required field" | |
| # ) | |
| # with gr.Row(): | |
| # gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # release_time = gr.Textbox(label="Time of Publish", placeholder="1970-01-01") | |
| # model_resolution = gr.Textbox(label="resolution", placeholder="Width x Height") | |
| # model_fps = gr.Textbox(label="model fps", placeholder="FPS(int)") | |
| # model_frame = gr.Textbox(label="model frame count", placeholder="INT") | |
| # model_video_length = gr.Textbox(label="model video length", placeholder="float(2.0)") | |
| # model_checkpoint = gr.Textbox(label="model checkpoint", placeholder="optional") | |
| # model_commit_id = gr.Textbox(label="github commit id", placeholder='main') | |
| # model_video_format = gr.Textbox(label="pipeline format", placeholder='mp4') | |
| # with gr.Column(): | |
| # input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary') | |
| # submit_button = gr.Button("Submit Eval") | |
| # submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False) | |
| # fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False) | |
| # submission_result = gr.Markdown() | |
| # # submit_button.click( | |
| # # add_new_eval, | |
| # # inputs = [ | |
| # # input_file, | |
| # # model_name_textbox, | |
| # # revision_name_textbox, | |
| # # model_link, | |
| # # team_name, | |
| # # contact_email, | |
| # # release_time, | |
| # # access_type, | |
| # # model_resolution, | |
| # # model_fps, | |
| # # model_frame, | |
| # # model_video_length, | |
| # # model_checkpoint, | |
| # # model_commit_id, | |
| # # model_video_format | |
| # # ], | |
| # # outputs=[submit_button, submit_succ_button, fail_textbox] | |
| # # ) | |
| # with gr.TabItem("🚀 [I2V]Submit here! ", elem_id="mvbench-i2v-tab-table", id=7): | |
| # gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # gr.Markdown("# ✉️✨ Submit your i2v model evaluation json file here!", elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # gr.Markdown("Here is a required field", elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # with gr.Column(): | |
| # model_name_textbox_i2v = gr.Textbox( | |
| # label="Model name", placeholder="Required field" | |
| # ) | |
| # revision_name_textbox_i2v = gr.Textbox( | |
| # label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line" | |
| # ) | |
| # access_type_i2v = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.") | |
| # with gr.Column(): | |
| # model_link_i2v = gr.Textbox( | |
| # label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed." | |
| # ) | |
| # team_name_i2v = gr.Textbox( | |
| # label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload" | |
| # ) | |
| # contact_email_i2v = gr.Textbox( | |
| # label="E-Mail(Will not be displayed)", placeholder="Required field" | |
| # ) | |
| # with gr.Row(): | |
| # gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # release_time_i2v = gr.Textbox(label="Time of Publish", placeholder="1970-01-01") | |
| # model_resolution_i2v = gr.Textbox(label="resolution", placeholder="Width x Height") | |
| # model_fps_i2v = gr.Textbox(label="model fps", placeholder="FPS(int)") | |
| # model_frame_i2v = gr.Textbox(label="model frame count", placeholder="INT") | |
| # model_video_length_i2v = gr.Textbox(label="model video length", placeholder="float(2.0)") | |
| # model_checkpoint_i2v = gr.Textbox(label="model checkpoint", placeholder="optional") | |
| # model_commit_id_i2v = gr.Textbox(label="github commit id", placeholder='main') | |
| # model_video_format_i2v = gr.Textbox(label="pipeline format", placeholder='mp4') | |
| # with gr.Column(): | |
| # input_file_i2v = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary') | |
| # submit_button_i2v = gr.Button("Submit Eval") | |
| # submit_succ_button_i2v = gr.Markdown("Submit Success! Please press refresh and retfurn to LeaderBoard!", visible=False) | |
| # fail_textbox_i2v = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False) | |
| # submission_result_i2v = gr.Markdown() | |
| # # submit_button_i2v.click( | |
| # # add_new_eval_i2v, | |
| # # inputs = [ | |
| # # input_file_i2v, | |
| # # model_name_textbox_i2v, | |
| # # revision_name_textbox_i2v, | |
| # # model_link_i2v, | |
| # # team_name_i2v, | |
| # # contact_email_i2v, | |
| # # release_time_i2v, | |
| # # access_type_i2v, | |
| # # model_resolution_i2v, | |
| # # model_fps_i2v, | |
| # # model_frame_i2v, | |
| # # model_video_length_i2v, | |
| # # model_checkpoint_i2v, | |
| # # model_commit_id_i2v, | |
| # # model_video_format_i2v | |
| # # ], | |
| # # outputs=[submit_button_i2v, submit_succ_button_i2v, fail_textbox_i2v] | |
| # # ) | |
| # def refresh_data(): | |
| # value1 = get_baseline_df() | |
| # return value1 | |
| # with gr.Row(): | |
| # data_run = gr.Button("Refresh") | |
| # data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component) | |
| block.launch() | |