implement submission

app.py

@@ -144,7 +144,7 @@ with demo:
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
             with gr.Row():
-                gr.Markdown("# Submission Form", elem_classes="markdown-text")
+                gr.Markdown("# Submission Form\nSubmitted files will be stored and made public.", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():

src/challenges/result_parsers.py

@@ -0,0 +1,15 @@
+import json
+
+
+def parse_challenge_result_dict(challenge_name: str, challenge_result_dict: dict) -> float:
+    """
+    Parse the challenge result dictionary and return the score.
+    Currently only reads the score stored in the dict. Will be updated to include score verification.
+    """
+    score_fields = ['score', 'accuracy']
+    
+    for field in score_fields:
+        if field in challenge_result_dict:
+            return challenge_result_dict[field]
+    
+    raise ValueError(f"Could not parse the score for {challenge_name}.")

src/envs.py

@@ -6,12 +6,11 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "PKU-Alignment" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/ProgressGym-LeaderBoard"
+DATA_REPO = f"{OWNER}/ProgressGym-LeaderBoardData"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/legacy/submit.py

@@ -0,0 +1,119 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )

src/submission/submit.py

@@ -1,9 +1,21 @@
 import json
 import os
+import re
 from datetime import datetime, timezone
 
+from src.challenges.result_parsers import parse_challenge_result_dict
+
+# email validity checker
+from email.utils import parseaddr
+
+# url validity checker
+from urllib.parse import urlparse
+
+# json parser
+from json.decoder import JSONDecodeError
+
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, DATA_REPO
 from src.submission.check_validity import (
     already_submitted_models,
     check_model_card,
@@ -11,109 +23,113 @@ from src.submission.check_validity import (
     is_model_on_hub,
 )
 
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
 def add_new_eval(
-    submission_file: str,
+    submission_file,
     algo_name: str,
     algo_info: str,
     algo_link: str,
     submitter_email: str,
 ):
-    return 'Success!'
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
+    return_str = 'Success! Your submission will soon be added to the leaderboard.'
+
+    # validate email and url
+    if not parseaddr(submitter_email):
+        return styled_error("Please enter a valid email address.")
+    
+    if algo_link.strip() and not urlparse(algo_link).scheme:
+        return styled_error("Please enter a valid URL.")
+    
+    # get file path
     try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
+        file_path: str = submission_file.name,
+        assert isinstance(file_path, str)
+    except:
+        if isinstance(submission_file, str):
+            file_path: str = submission_file
+        else:
+            return styled_error("Invalid submission file: File path not found.")
+    
+    # parse the submission file
     try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
+        submission_data = json.loads(file_path)
+    except JSONDecodeError:
+        return styled_error("Invalid submission file: JSON parsing failed.")
+    
+    try:
+        assert isinstance(submission_data, dict)
+        submission_data_content = list(submission_data.items())
+        assert len(submission_data_content) == 1
+        results_per_challenge = submission_data_content[0][1]
+        assert isinstance(results_per_challenge, dict)
+        assert all(isinstance(challenge, str) for challenge in results_per_challenge.keys())
+        assert all(isinstance(result, dict) for result in results_per_challenge.values())
+    except (AssertionError, KeyError):
+        return styled_error("Invalid submission file: Incorrect organization of the JSON file.")
+    
+    # format the algo name
+    algo_name = algo_name.strip()
+    algo_name_filename = re.sub(r"[^a-zA-Z0-9]+", "-", algo_name).lower()
+    timestamp_filename = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")
+    
+    print("Uploading submission file")
+    API.upload_file(
+        path_or_fileobj=file_path,
+        path_in_repo=f'upload_history/{algo_name_filename}/{timestamp_filename}.json',
+        repo_id=DATA_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {algo_name} to eval queue by {submitter_email} at {timestamp_filename}",
+    )
 
+    # Construct entry in the master table
     eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
+        "name": algo_name,
+        "id": algo_name_filename,
+        "info": algo_info,
+        "link": algo_link,
+        "email": submitter_email,
+        "update_timestamp": timestamp_filename,
     }
-
+    
+    for challenge, result in results_per_challenge:
+        try:
+            parsed_result: float = parse_challenge_result_dict(challenge, result)
+            assert isinstance(parsed_result, float)
+        except:
+            return styled_error(f"Could not parse the score for {challenge}.")
+        
+        eval_entry[challenge] = parsed_result
+    
+    # Get content of the master table from DATA_REPO
+    try:
+        master_table = {}
+        if API.file_exists(DATA_REPO, "master_table.json"):
+            API.hf_hub_download(DATA_REPO, "master_table.json", EVAL_REQUESTS_PATH, force_download=True)
+            with open(f"{EVAL_REQUESTS_PATH}/master_table.json", "r") as f:
+                master_table = json.load(f)
+        else:
+            print("No master table found. Will create a new one.")
+    except:
+        return styled_error("Could not get the master table from the data repository.")
+    
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
+    if algo_name_filename in master_table:
+        return_str += ' An existing submission with the same name has been found. Your submission will be used to update the existing one.'
+        master_table[algo_name_filename].update(eval_entry)
+    else:
+        print("Creating eval entry")
+        master_table[algo_name_filename] = eval_entry
+
+    # Save the updated master table
+    with open(f"./master_table.json", "w") as f:
+        f.write(json.dumps(master_table))
+    
+    print("Uploading master table")
     API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
+        path_or_fileobj="./master_table.json",
+        path_in_repo="master_table.json",
+        repo_id=DATA_REPO,
         repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
+        commit_message=f"Update master table with {algo_name} by {submitter_email} at {timestamp_filename}",
     )
 
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
+    return styled_message(return_str)