Upload 2 files

app.py

@@ -23,296 +23,249 @@ def upload_file(files):
     return file_paths
 
 
-def add_new_eval_i2v(
+def add_new_eval(
     input_file,
-    model_name_textbox: str,
-    revision_name_textbox: str,
-    model_link: str,
-    team_name: str,
-    contact_email: str,
-    access_type: str,
-    model_publish: str,
-    model_resolution: str,
-    model_fps: str,
-    model_frame: str,
-    model_video_length: str,
-    model_checkpoint: str,
-    model_commit_id: str,
-    model_video_format: str
+    model_type,
+    model_name_textbox,
+    model_ability,
+    revision_name_textbox,
+    access_type,
+    model_link,
+    team_name,
+    contact_email,
+    release_time,
+    model_resolution,
+    model_length,
+    model_fps,
+    model_frame,
+    model_link_optional,
 ):
-    COLNAME2KEY={
-        "Video-Text Camera Motion":"camera_motion",
-        "Video-Image Subject Consistency": "i2v_subject",
-        "Video-Image Background Consistency": "i2v_background",
-        "Subject Consistency": "subject_consistency",
-        "Background Consistency": "background_consistency",
-        "Motion Smoothness": "motion_smoothness",
-        "Dynamic Degree": "dynamic_degree",
-        "Aesthetic Quality": "aesthetic_quality",
-        "Imaging Quality": "imaging_quality",
-        "Temporal Flickering": "temporal_flickering"
-        }
     if input_file is None:
         return "Error! Empty file!"
     if  model_link == '' or model_name_textbox == '' or contact_email == '':
         return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
     
-    upload_content = input_file
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
-    filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"{model_name_textbox}_{model_type}_{model_ability}_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}"
     
     now = datetime.datetime.now()
-    update_time = now.strftime("%Y-%m-%d")  # Capture update time
-    with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
-        f.write(input_file)
-    # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
+    update_time = now.strftime("%Y.%m.%d")  # Capture update time
 
-    csv_data = pd.read_csv(I2V_DIR)
+    csv_data = pd.read_csv(CSV_DIR)
 
     if revision_name_textbox == '':
         col = csv_data.shape[0]
         model_name = model_name_textbox.replace(',',' ')
     else:
         model_name = revision_name_textbox.replace(',',' ')
-        model_name_list = csv_data['Model Name (clickable)']
+        model_name_list = csv_data['Model Name']
         name_list = [name.split(']')[0][1:] for name in model_name_list]
         if revision_name_textbox not in name_list:
             col = csv_data.shape[0]
         else:
             col = name_list.index(revision_name_textbox)    
+            if csv_data[col]['Sampled by'] == "WorldScore" or csv_data[col]['Evaluated by'] == "WorldScore":
+                return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+
     if model_link == '':
         model_name = model_name  # no url
     else:
         model_name = '[' + model_name + '](' + model_link + ')'
 
-    os.makedirs(filename, exist_ok=True)
-    with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
-        zip_ref.extractall(filename)
-
-    upload_data = {}
-    for file in os.listdir(filename):
-        if file.startswith('.') or file.startswith('__'):
-            print(f"Skip the file: {file}")
-            continue
-        cur_file = os.path.join(filename, file)
-        if os.path.isdir(cur_file):
-            for subfile in os.listdir(cur_file):
-                if subfile.endswith(".json"):
-                    with open(os.path.join(cur_file, subfile)) as ff:
-                        cur_json = json.load(ff)
-                        print(file, type(cur_json))
-                        if isinstance(cur_json, dict):
-                            print(cur_json.keys())
-                            for key in cur_json:
-                                upload_data[key] = cur_json[key][0]
-                                print(f"{key}:{cur_json[key][0]}")
-        elif cur_file.endswith('json'):
-            with open(cur_file) as ff:
-                cur_json = json.load(ff)
-                print(file, type(cur_json))
-                if isinstance(cur_json, dict):
-                    print(cur_json.keys())
-                    for key in cur_json:
-                        upload_data[key] = cur_json[key][0]
-                        print(f"{key}:{cur_json[key][0]}")
-    # add new data
-    new_data = [model_name]
-    print('upload_data:', upload_data)
-    I2V_HEAD= ["Video-Text Camera Motion",
-    "Video-Image Subject Consistency",
-    "Video-Image Background Consistency",
-    "Subject Consistency",
-    "Background Consistency",
-    "Temporal Flickering",
-    "Motion Smoothness",
-    "Dynamic Degree",
-    "Aesthetic Quality",
-    "Imaging Quality" ]
-    for key in I2V_HEAD :
-        sub_key = COLNAME2KEY[key]
-        if sub_key in upload_data:
-            new_data.append(upload_data[sub_key])
+    if model_link_optional == '':
+        model_link_optional = model_link
+        
+    try:
+        with open(input_file, 'r') as f:
+            upload_data = json.load(f)
+    
+        # add new data
+        print('upload_data:', upload_data)
+        new_data = [model_type, model_name, model_ability]
+        if team_name == '':
+            new_data.append(model_name)
+            new_data.append(model_name)
         else:
-            new_data.append(0)
-    if team_name =='' or 'vbench' in team_name.lower():
-        new_data.append("User Upload")
-    else:
-        new_data.append(team_name)
-
-    new_data.append(contact_email.replace(',',' and ')) # Add contact email [private]
-    new_data.append(update_time)  # Add the update time
-    new_data.append(team_name)
-    new_data.append(access_type)
-
-    csv_data.loc[col] = new_data
-    csv_data = csv_data.to_csv(I2V_DIR , index=False)
-    with open(INFO_DIR,'a') as f:
-        f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n")
-    submission_repo.push_to_hub()
-    print("success update", model_name)
-    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
-
+            new_data.append(team_name)
+            new_data.append(team_name)
+        new_data.append(access_type)
+        new_data.append(update_time)
+        
+        for key in TASK_INFO:
+            value = COLNAME2KEY[key]
+            if value in upload_data:
+                new_data.append(upload_data[value])
+            else:
+                return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+        
+        
+        csv_data.loc[col] = new_data
+        csv_data = csv_data.to_csv(CSV_DIR, index=False)
+        with open(INFO_DIR,'a') as f:
+            f.write(f"{model_type.replace(',', ' ')}\t{model_name.replace(',', ' ')}\t{model_ability.replace(',', ' ')}\t{model_resolution.replace(',', ' ')}\t{model_length.replace(',', ' ')}\t{model_fps.replace(',', ' ')}\t{model_frame.replace(',', ' ')}\t{model_link_optional.replace(',', ' ')}\t{contact_email.replace(',', ' ')}\n")
+        submission_repo.push_to_hub()
+        print("success update", model_name)
+        return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+    except:
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+    
+    
 def get_baseline_df():
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
     df = pd.read_csv(CSV_DIR)
-    df = df.sort_values(by=DEFAULT_INFO[0], ascending=False)
-
-    # Add this line to display the results of all model types by default
-    df = df[df['Model Type'].isin(model_type_filter.value)]
-    # Add this line to display the results of both abilities by default
-    df = df[df['Ability'].isin(ability_filter.value)]    
-    return df
-
-def get_all_df(dir=CSV_DIR):
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
-    df = pd.read_csv(dir)
+    df = df.sort_values(by=DEFAULT_INFO[0], ascending=False)  
     return df
 
 
-block = gr.Blocks()
+block = gr.Blocks(css="""
+    .container {
+        max-width: 80%;
+        margin: auto;
+    }
+""")
 with block:
-    gr.Markdown(
-        LEADERBORAD_INTRODUCTION
-    )
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        # Table 0
-        with gr.TabItem("🏅 WorldScore Benchmark", elem_id="worldscore-tab-table", id=1):
-            with gr.Column():
-                model_type_filter = gr.CheckboxGroup(
-                    choices=MODEL_TYPE,
-                    value=DEFAULT_MODEL_TYPE,
-                    label="Model Type",
-                    interactive=True
-                )
-                ability_filter = gr.CheckboxGroup(
-                    choices=ABILITY,
-                    value=DEFAULT_ABILITY,
-                    label="Ability",
-                    interactive=True
-                )
-                
-            data_component = gr.components.Dataframe(
-                column_widths="auto",  # Automatically adjusts column widths
-                value=get_baseline_df(), 
-                headers=COLUMN_NAMES,
-                type="pandas", 
-                datatype=DATA_TITILE_TYPE,
-                interactive=False,
-                visible=True,
-            )
-    
-            def on_filter_change(model_types, abilities):
-                df = get_baseline_df()
-                # Filter by selected model types
-                df = df[df['Model Type'].isin(model_types)]
-                # Filter by selected abilities 
-                df = df[df['Ability'].isin(abilities)]
-                return gr.Dataframe(
-                    column_widths="auto",  # Automatically adjusts column widths
-                    value=df,
+    with gr.Column(elem_classes="container"):
+        gr.Markdown(
+            LEADERBORAD_INTRODUCTION
+        )
+        with gr.Tabs(elem_classes="tab-buttons") as tabs:
+            # Table 0
+            with gr.TabItem("🏅 WorldScore Benchmark", elem_id="worldscore-tab-table", id=0):
+                with gr.Column():
+                    model_type_filter = gr.CheckboxGroup(
+                        choices=MODEL_TYPE,
+                        value=DEFAULT_MODEL_TYPE,
+                        label="Model Type",
+                        interactive=True
+                    )
+                    ability_filter = gr.CheckboxGroup(
+                        choices=ABILITY,
+                        value=DEFAULT_ABILITY,
+                        label="Ability",
+                        interactive=True
+                    )
+                    
+                data_component = gr.components.Dataframe(
+                    value=get_baseline_df(),
                     headers=COLUMN_NAMES,
-                    type="pandas",
+                    type="pandas", 
                     datatype=DATA_TITILE_TYPE,
                     interactive=False,
-                    visible=True
+                    visible=True,
                 )
+        
+                def on_filter_change(model_types, abilities):
+                    df = get_baseline_df()
+                    # Filter by selected model types
+                    df = df[df['Model Type'].isin(model_types)]
+                    # Filter by selected abilities 
+                    df = df[df['Ability'].isin(abilities)]
+                    return gr.Dataframe(
+                        value=df,
+                        headers=COLUMN_NAMES,
+                        type="pandas",
+                        datatype=DATA_TITILE_TYPE,
+                        interactive=False,
+                        visible=True
+                    )
 
-            model_type_filter.change(
-                fn=on_filter_change,
-                inputs=[model_type_filter, ability_filter],
-                outputs=data_component
-            )
+                model_type_filter.change(
+                    fn=on_filter_change,
+                    inputs=[model_type_filter, ability_filter],
+                    outputs=data_component
+                )
 
-            ability_filter.change(
-                fn=on_filter_change,
-                inputs=[model_type_filter, ability_filter], 
-                outputs=data_component
-            )
-                
-        with gr.TabItem("🚀 Submit here! ", elem_id="submit-tab-table", id=7):
+                ability_filter.change(
+                    fn=on_filter_change,
+                    inputs=[model_type_filter, ability_filter], 
+                    outputs=data_component
+                )
+                    
+            with gr.TabItem("🚀 Submit here! ", elem_id="submit-tab-table", id=1):
 
-            with gr.Row():
-                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+                with gr.Row():
+                    gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
-            with gr.Row():
-                gr.Markdown("# Submit your world generation model evaluation json file here!", elem_classes="markdown-text")
+                with gr.Row():
+                    gr.Markdown("# Submit your evaluation json file here!", elem_classes="markdown-text")
 
-            with gr.Row():
-                gr.Markdown("Here is a required field", elem_classes="markdown-text")
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox_i2v = gr.Textbox(
-                        label="Model name", placeholder="Required field"
+                with gr.Row():
+                    gr.Markdown("Here is a required field", elem_classes="markdown-text")
+                with gr.Row():
+                    with gr.Column():
+                        model_type = gr.Dropdown(["Video", "3D", "4D"], label="Model Type")
+                        model_name_textbox = gr.Textbox(
+                            label="Model name", placeholder="Your model name"
+                            )
+                        model_ability = gr.Dropdown(["I2V", "T2V"], label="Model Ability")
+                        revision_name_textbox = gr.Textbox(
+                            label="Revision Model Name (Optional)", placeholder="If you need to update the previous submissions, please fill in this line"
                         )
-                    revision_name_textbox_i2v = gr.Textbox(
-                        label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
-                    )
-                    access_type_i2v = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Please select the way user can access your model. You can update the content by revision_name, or contact the VBench Team.")
-
 
+                    with gr.Column():
+                        access_type = gr.Dropdown(["Open Source", "Ready to Open Source", "API", "Close"], label="Access")
+                        model_link = gr.Textbox(
+                            label="Link (Website/Paper Link/Github/HuggingFace)", placeholder="If filling in the wrong information, your results may be removed."
+                        )
+                        team_name = gr.Textbox(
+                            label="Your Team Name", placeholder="If left blank, it will be your model name"
+                        )
+                        contact_email = gr.Textbox(
+                            label="E-Mail (Will not be displayed)", placeholder="Contact email"
+                        )
+                with gr.Row():
+                    gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
+                with gr.Row():
+                        release_time = gr.Textbox(label="Version", placeholder="2025.03.29")
+                        model_resolution = gr.Textbox(label="Resolution", placeholder="WidthxHeight")
+                        model_length = gr.Textbox(label="Video Length (s)", placeholder="float")
+                        model_fps = gr.Textbox(label="FPS", placeholder="int")
+                        model_frame = gr.Textbox(label="Frame Number", placeholder="int")
+                        model_link_optional = gr.Textbox(label="Link", placeholder='optional')
                 with gr.Column():
-                    model_link_i2v = gr.Textbox(
-                        label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
+                    input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
+                    submit_button = gr.Button("Submit Eval")
+                    submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to WorldScore Benchmark!", visible=False)
+                    fail_textbox = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `E-mail` are filled in correctly and the uploaded json file is valid.</span>', elem_classes="markdown-text",visible=False)
+                    
+        
+                    submission_result = gr.Markdown()
+                    submit_button.click(
+                        add_new_eval,
+                        inputs = [
+                            input_file,
+                            model_type,
+                            model_name_textbox,
+                            model_ability,
+                            revision_name_textbox,
+                            access_type,
+                            model_link,
+                            team_name,
+                            contact_email,
+                            release_time,
+                            model_resolution,
+                            model_length,
+                            model_fps,
+                            model_frame,
+                            model_link_optional,
+                        ],
+                        outputs=[submit_button, submit_succ_button, fail_textbox]
                     )
-                    team_name_i2v = gr.Textbox(
-                        label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
-                    )
-                    contact_email_i2v = gr.Textbox(
-                        label="E-Mail(Will not be displayed)", placeholder="Required field"
-                    )
-            with gr.Row():
-                gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
-            with gr.Row():
-                    release_time_i2v = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
-                    model_resolution_i2v = gr.Textbox(label="resolution", placeholder="Width x Height")
-                    model_fps_i2v = gr.Textbox(label="model fps", placeholder="FPS(int)")
-                    model_frame_i2v = gr.Textbox(label="model frame count", placeholder="INT")
-                    model_video_length_i2v = gr.Textbox(label="model video length", placeholder="float(2.0)")
-                    model_checkpoint_i2v = gr.Textbox(label="model checkpoint", placeholder="optional")
-                    model_commit_id_i2v = gr.Textbox(label="github commit id", placeholder='main')
-                    model_video_format_i2v = gr.Textbox(label="pipeline format", placeholder='mp4')
-            with gr.Column():
-                input_file_i2v = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
-                submit_button_i2v = gr.Button("Submit Eval")
-                submit_succ_button_i2v = gr.Markdown("Submit Success! Please press refresh and retfurn to LeaderBoard!", visible=False)
-                fail_textbox_i2v = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
-                
-    
-                submission_result_i2v = gr.Markdown()
-                # submit_button_i2v.click(
-                #     add_new_eval_i2v,
-                #     inputs = [
-                #         input_file_i2v,
-                #         model_name_textbox_i2v,
-                #         revision_name_textbox_i2v,
-                #         model_link_i2v,
-                #         team_name_i2v,
-                #         contact_email_i2v,
-                #         release_time_i2v,
-                #         access_type_i2v,
-                #         model_resolution_i2v,
-                #         model_fps_i2v,
-                #         model_frame_i2v,
-                #         model_video_length_i2v,
-                #         model_checkpoint_i2v,
-                #         model_commit_id_i2v,
-                #         model_video_format_i2v
-                #     ],
-                #     outputs=[submit_button_i2v, submit_succ_button_i2v, fail_textbox_i2v]
-                # )
-
 
 
-    # def refresh_data():
-    #     value1 = get_baseline_df()
-    #     return value1
+    def refresh_data():
+        value = get_baseline_df()
+        return value, DEFAULT_MODEL_TYPE, DEFAULT_ABILITY
 
-    # with gr.Row():
-    #     data_run = gr.Button("Refresh")
-    #     data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component)
+    with gr.Row(elem_classes="container"):
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            refresh_data, 
+            inputs=[], 
+            outputs=[data_component, model_type_filter, ability_filter]
+        )
 
 
 block.launch()

constants.py

@@ -48,6 +48,20 @@ MODEL_INFO = [
     "Date",
 ]
 
+COLNAME2KEY = {
+    "WorldScore-Static":"WorldScore-Static",
+    "WorldScore-Dynamic":"WorldScore-Dynamic",
+    "Camera Control":"camera_control",
+    "Object Control":"object_control",
+    "Content Alignment":"content_alignment",
+    "3D Consistency":"3d_consistency",
+    "Photometric Consistency":"photometric_consistency",
+    "Style Consistency":"style_consistency",
+    "Subjective Quality":"subjective_quality",
+    "Motion Accuracy":"motion_accuracy",
+    "Motion Magnitude":"motion_magnitude",
+    "Motion Smoothness":"motion_smoothness",
+}
 DATA_TITILE_TYPE = ['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
 
 SUBMISSION_NAME = "worldscore_leaderboard_submission"
@@ -58,16 +72,9 @@ INFO_DIR = "./worldscore_leaderboard_submission/model_info.csv"
 COLUMN_NAMES = MODEL_INFO + TASK_INFO
 
 LEADERBORAD_INTRODUCTION = """# WorldScore Leaderboard
+    [Paper](https://arxiv.org/abs/2503.15669) | [Website](https://haoyi-duan.github.io/WorldScore/) | [Code](https://github.com/haoyi-duan/WorldScore) | [Dataset](https://huggingface.co/datasets/Howieeeee/WorldScore)
     
     🏆 Welcome to the leaderboard of **WorldScore**, the first unified evaluation benchmark for world generation.  
-    <div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
-    <a href=''><img src='https://img.shields.io/badge/cs.CV-Paper-b31b1b?logo=arxiv&logoColor=red'></a>
-    <a href='https://haoyi-duan.github.io/WorldScore/'><img src='https://img.shields.io/badge/WorldScore-Website-green?logo=googlechrome&logoColor=green'></a>
-    <a href='https://github.com/haoyi-duan/WorldScore'><img src='https://img.shields.io/badge/WorldScore-Code-blue?logo=github&logoColor=blue'></a>
-    <a href='https://huggingface.co/datasets/Howieeeee/WorldScore'><img src='https://img.shields.io/badge/WordScore-Dataset-yellow?logo=huggingface&logoColor=yellow'></a>
-    </div>
-    
-    [Paper](https://arxiv.org/abs/2503.15669) | [Website](https://haoyi-duan.github.io/WorldScore/) | [Code](https://github.com/haoyi-duan/WorldScore) | [Dataset](https://huggingface.co/datasets/Howieeeee/WorldScore)
     
     **Join Leaderboard**: Please see the [instructions](https://github.com/Vchitect/VBench/tree/master?tab=readme-ov-file#trophy-leaderboard) for 3 options to participate. One option is to follow [VBench Usage info](https://github.com/Vchitect/VBench?tab=readme-ov-file#usage), and upload the generated `result.json` file here. After clicking the `Submit here!` button, click the `Refresh` button.
     """