submodule + versioning

.gitignore

@@ -43,3 +43,9 @@ eval-queue/
 eval-results/
 eval-queue-bk/
 eval-results-bk/
+
+# Data files
+data/
+
+# Versioned leaderboard files
+data/leaderboard_v*.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "guard-bench-submodule"]
+	path = guard-bench-submodule
+	url = https://github.com/whitecircle-ai/guard-bench.git

app.py

@@ -51,10 +51,14 @@ logger = logging.getLogger(__name__)
 # Ensure data directory exists
 os.makedirs(DATA_PATH, exist_ok=True)
 
+# Available benchmark versions
+BENCHMARK_VERSIONS = ["v0"]
+CURRENT_VERSION = "v0"
+
 # Initialize leaderboard data
 try:
     logger.info("Initializing leaderboard data...")
-    LEADERBOARD_DF = get_leaderboard_df()
+    LEADERBOARD_DF = get_leaderboard_df(version=CURRENT_VERSION)
     logger.info(f"Loaded leaderboard with {len(LEADERBOARD_DF)} entries")
 except Exception as e:
     logger.error(f"Error loading leaderboard data: {e}")
@@ -70,7 +74,7 @@ def init_leaderboard(dataframe):
         columns = [getattr(GUARDBENCH_COLUMN, col).name for col in DISPLAY_COLS]
         dataframe = pd.DataFrame(columns=columns)
         logger.warning("Initializing empty leaderboard")
-    
+
     return Leaderboard(
         value=dataframe,
         datatype=[getattr(GUARDBENCH_COLUMN, col).type for col in DISPLAY_COLS],
@@ -79,7 +83,7 @@ def init_leaderboard(dataframe):
             cant_deselect=[getattr(GUARDBENCH_COLUMN, col).name for col in NEVER_HIDDEN_COLS],
             label="Select Columns to Display:",
         ),
-        search_columns=[GUARDBENCH_COLUMN.model.name],
+        search_columns=[GUARDBENCH_COLUMN.model_name.name],
         hide_columns=[getattr(GUARDBENCH_COLUMN, col).name for col in HIDDEN_COLS],
         filter_columns=[
             ColumnFilter(GUARDBENCH_COLUMN.model_type.name, type="checkboxgroup", label="Model types"),
@@ -95,23 +99,24 @@ def submit_results(
     precision: str,
     weight_type: str,
     model_type: str,
-    submission_file: tempfile._TemporaryFileWrapper
+    submission_file: tempfile._TemporaryFileWrapper,
+    version: str
 ):
     """
     Handle submission of results with model metadata.
     """
     if submission_file is None:
         return styled_error("No submission file provided")
-    
+
     if not model_name:
         return styled_error("Model name is required")
-    
+
     if not model_type:
         return styled_error("Please select a model type")
-    
+
     file_path = submission_file.name
     logger.info(f"Received submission for model {model_name}: {file_path}")
-    
+
     # Add metadata to the submission
     metadata = {
         "model_name": model_name,
@@ -119,35 +124,46 @@ def submit_results(
         "revision": revision if revision else "main",
         "precision": precision,
         "weight_type": weight_type,
-        "model_type": model_type
+        "model_type": model_type,
+        "version": version
     }
-    
+
     # Process the submission
-    result = process_submission(file_path, metadata)
-    
+    result = process_submission(file_path, metadata, version=version)
+
     # Refresh the leaderboard data
     global LEADERBOARD_DF
     try:
-        logger.info("Refreshing leaderboard data after submission...")
-        LEADERBOARD_DF = get_leaderboard_df()
+        logger.info(f"Refreshing leaderboard data after submission for version {version}...")
+        LEADERBOARD_DF = get_leaderboard_df(version=version)
         logger.info("Refreshed leaderboard data after submission")
     except Exception as e:
         logger.error(f"Error refreshing leaderboard data: {e}")
-    
+
     return result
 
 
-def refresh_data():
+def refresh_data(version=CURRENT_VERSION):
     """
     Refresh the leaderboard data from HuggingFace.
     """
     global LEADERBOARD_DF
     try:
-        logger.info("Performing scheduled refresh of leaderboard data...")
-        LEADERBOARD_DF = get_leaderboard_df()
+        logger.info(f"Performing scheduled refresh of leaderboard data for version {version}...")
+        LEADERBOARD_DF = get_leaderboard_df(version=version)
         logger.info("Scheduled refresh of leaderboard data completed")
     except Exception as e:
         logger.error(f"Error in scheduled refresh: {e}")
+    return LEADERBOARD_DF
+
+
+def update_leaderboards(version):
+    """
+    Update all leaderboard components with data for the selected version.
+    """
+    new_df = get_leaderboard_df(version=version)
+    category_dfs = [get_category_leaderboard_df(category, version=version) for category in CATEGORIES]
+    return [init_leaderboard(new_df)] + [init_leaderboard(df) for df in category_dfs]
 
 
 # Create Gradio app
@@ -155,43 +171,54 @@ demo = gr.Blocks(css=custom_css)
 
 with demo:
     gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+        with gr.Column(scale=1):
+            version_selector = gr.Dropdown(
+                choices=BENCHMARK_VERSIONS,
+                label="Benchmark Version",
+                value=CURRENT_VERSION,
+                interactive=True,
+                elem_classes="version-selector"
+            )
+
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 Leaderboard", elem_id="guardbench-leaderboard-tab", id=0):
             refresh_button = gr.Button("Refresh Leaderboard")
-            
+
             # Create tabs for each category
             with gr.Tabs(elem_classes="category-tabs") as category_tabs:
                 # First tab for average metrics across all categories
                 with gr.TabItem("📊 Overall Performance", elem_id="overall-tab"):
                     leaderboard = init_leaderboard(LEADERBOARD_DF)
-                
+
                 # Create a tab for each category
                 for category in CATEGORIES:
                     with gr.TabItem(f"{category}", elem_id=f"category-{category.lower().replace(' ', '-')}-tab"):
-                        category_df = get_category_leaderboard_df(category)
+                        category_df = get_category_leaderboard_df(category, version=CURRENT_VERSION)
                         category_leaderboard = init_leaderboard(category_df)
-            
+
             # Refresh button functionality
             refresh_button.click(
                 fn=lambda: [
-                    init_leaderboard(get_leaderboard_df()),
-                    *[init_leaderboard(get_category_leaderboard_df(category)) for category in CATEGORIES]
+                    init_leaderboard(get_leaderboard_df(version=version_selector.value)),
+                    *[init_leaderboard(get_category_leaderboard_df(category, version=version_selector.value)) for category in CATEGORIES]
                 ],
                 inputs=[],
                 outputs=[leaderboard] + [category_tabs.children[i].children[0] for i in range(1, len(CATEGORIES) + 1)]
             )
-        
+
         with gr.TabItem("📝 About", elem_id="guardbench-about-tab", id=1):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-        
+
         with gr.TabItem("🚀 Submit", elem_id="guardbench-submit-tab", id=2):
             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-            
+
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your results here!", elem_classes="markdown-text")
-            
+
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
@@ -203,33 +230,33 @@ with demo:
                         value=None,
                         interactive=True,
                     )
-                
+
                 with gr.Column():
                     precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        choices=[i.name for i in Precision if i != Precision.Unknown],
                         label="Precision",
                         multiselect=False,
                         value="float16",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
+                        choices=[i.name for i in WeightType],
                         label="Weights type",
                         multiselect=False,
                         value="Original",
                         interactive=True,
                     )
                     base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-            
+
             with gr.Row():
                 file_input = gr.File(
-                    label="Upload JSONL Results File", 
+                    label="Upload JSONL Results File",
                     file_types=[".jsonl"]
                 )
-            
+
             submit_button = gr.Button("Submit Results")
             result_output = gr.Markdown()
-            
+
             submit_button.click(
                 fn=submit_results,
                 inputs=[
@@ -239,11 +266,19 @@ with demo:
                     precision,
                     weight_type,
                     model_type,
-                    file_input
+                    file_input,
+                    version_selector
                 ],
                 outputs=result_output
             )
-    
+
+    # Version selector functionality
+    version_selector.change(
+        fn=update_leaderboards,
+        inputs=[version_selector],
+        outputs=[leaderboard] + [category_tabs.children[i].children[0] for i in range(1, len(CATEGORIES) + 1)]
+    )
+
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -253,29 +288,21 @@ with demo:
                 elem_id="citation-button",
                 show_copy_button=True,
             )
-        
+
         with gr.Accordion("ℹ️ Dataset Information", open=False):
             dataset_info = gr.Markdown(f"""
             ## Dataset Information
-            
+
             Results are stored in the HuggingFace dataset: [{RESULTS_DATASET_ID}](https://huggingface.co/datasets/{RESULTS_DATASET_ID})
-            
+
             Last updated: {pd.Timestamp.now().strftime("%Y-%m-%d %H:%M:%S UTC")}
             """)
 
-# Set up scheduler to refresh data periodically
 scheduler = BackgroundScheduler()
-scheduler.add_job(refresh_data, 'interval', minutes=30)
+scheduler.add_job(lambda: refresh_data(version=CURRENT_VERSION), 'interval', minutes=30)
 scheduler.start()
 
 # Launch the app
 if __name__ == "__main__":
-    # Set up authentication if credentials are provided
-    if not ADMIN_USERNAME or not ADMIN_PASSWORD:
-        logger.warning("Admin username or password not set. Running without authentication.")
-        auth = None
-    else:
-        auth = (ADMIN_USERNAME, ADMIN_PASSWORD)
-    
-    # Launch the app
-    demo.launch(server_name="0.0.0.0", server_port=7860, auth=auth)
+
+    demo.launch(server_name="0.0.0.0", server_port=7860, share=True)

guard-bench-submodule

@@ -0,0 +1 @@
+Subproject commit 0a9f48bcedd0ccb6b5cf59ff7ed1186e32a5dc17

src/display/css_html_js.py

@@ -43,4 +43,22 @@ custom_css = """
     text-decoration: underline;
     color: #1976D2;
 }
+
+.version-selector {
+    margin-top: 10px;
+    padding: 5px;
+    border: 1px solid #e0e0e0;
+    border-radius: 5px;
+    background-color: #f9f9f9;
+}
+
+.version-selector label {
+    font-weight: bold;
+    color: #2196F3;
+}
+
+.version-selector select {
+    border-color: #2196F3;
+    border-radius: 4px;
+}
 """

src/display/utils.py

@@ -36,12 +36,19 @@ class Precision(Enum):
     int8 = auto()
     int4 = auto()
 
+    def __str__(self):
+        """String representation of the precision type."""
+        return self.name
+
 
 class WeightType(Enum):
     """Model weight types."""
     Original = auto()
     Delta = auto()
     Adapter = auto()
+    def __str__(self):
+        """String representation of the weight type."""
+        return self.name
 
 
 @dataclass
@@ -58,19 +65,19 @@ class ColumnInfo:
 @dataclass
 class GuardBenchColumn:
     """Columns for the GuardBench leaderboard."""
-    model: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+    model_name: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="model_name",
         display_name="Model",
         never_hidden=True,
         displayed_by_default=True
     ))
-    
+
     model_type: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="model_type",
         display_name="Type",
         displayed_by_default=True
     ))
-    
+
     # Metrics for all categories
     default_prompts_f1: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="default_prompts_f1",
@@ -78,28 +85,28 @@ class GuardBenchColumn:
         type="number",
         displayed_by_default=True
     ))
-    
+
     jailbreaked_prompts_f1: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="jailbreaked_prompts_f1",
         display_name="Jailbreaked Prompts F1",
         type="number",
         displayed_by_default=True
     ))
-    
+
     default_answers_f1: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="default_answers_f1",
         display_name="Default Answers F1",
         type="number",
         displayed_by_default=True
     ))
-    
+
     jailbreaked_answers_f1: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="jailbreaked_answers_f1",
         display_name="Jailbreaked Answers F1",
         type="number",
         displayed_by_default=True
     ))
-    
+
     # Average metrics
     average_f1: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="average_f1",
@@ -108,21 +115,21 @@ class GuardBenchColumn:
         displayed_by_default=True,
         never_hidden=True
     ))
-    
+
     average_recall: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="average_recall",
         display_name="Average Recall",
         type="number",
         displayed_by_default=False
     ))
-    
+
     average_precision: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="average_precision",
         display_name="Average Precision",
         type="number",
         displayed_by_default=False
     ))
-    
+
     # Additional metadata
     submission_date: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="submission_date",
@@ -136,13 +143,13 @@ GUARDBENCH_COLUMN = GuardBenchColumn()
 
 # Extract column lists for different views
 COLS = [f.name for f in fields(GUARDBENCH_COLUMN)]
-DISPLAY_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN) 
+DISPLAY_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN)
                 if getattr(GUARDBENCH_COLUMN, f.name).displayed_by_default]
-METRIC_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN) 
+METRIC_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN)
                if getattr(GUARDBENCH_COLUMN, f.name).type == "number"]
-HIDDEN_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN) 
+HIDDEN_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN)
                if getattr(GUARDBENCH_COLUMN, f.name).hidden]
-NEVER_HIDDEN_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN) 
+NEVER_HIDDEN_COLS = [getattr(GUARDBENCH_COLUMN, f.name).name for f in fields(GUARDBENCH_COLUMN)
                      if getattr(GUARDBENCH_COLUMN, f.name).never_hidden]
 
 # Categories in GuardBench

src/leaderboard/processor.py

@@ -16,11 +16,21 @@ def load_leaderboard_data(file_path: str) -> Dict:
     Load the leaderboard data from a JSON file.
     """
     if not os.path.exists(file_path):
-        return {"entries": [], "last_updated": datetime.now().isoformat()}
-    
+        version = "v0"
+        if "_v" in file_path:
+            version = file_path.split("_")[-1].split(".")[0]
+        return {"entries": [], "last_updated": datetime.now().isoformat(), "version": version}
+
     with open(file_path, 'r') as f:
         data = json.load(f)
-    
+
+    # Ensure version field exists
+    if "version" not in data:
+        version = "v0"
+        if "_v" in file_path:
+            version = file_path.split("_")[-1].split(".")[0]
+        data["version"] = version
+
     return data
 
 
@@ -30,10 +40,17 @@ def save_leaderboard_data(data: Dict, file_path: str) -> None:
     """
     # Ensure the directory exists
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
-    
+
     # Update the last_updated timestamp
     data["last_updated"] = datetime.now().isoformat()
-    
+
+    # Ensure version is set
+    if "version" not in data:
+        version = "v0"
+        if "_v" in file_path:
+            version = file_path.split("_")[-1].split(".")[0]
+        data["version"] = version
+
     with open(file_path, 'w') as f:
         json.dump(data, f, indent=2)
 
@@ -43,26 +60,32 @@ def process_submission(submission_data: List[Dict]) -> List[Dict]:
     Process submission data and convert it to leaderboard entries.
     """
     entries = []
-    
+
     for item in submission_data:
         # Create a new entry for the leaderboard
         entry = {
             "model_name": item.get("model_name", "Unknown Model"),
             "per_category_metrics": {},
             "avg_metrics": {},
-            "submission_date": datetime.now().isoformat()
+            "submission_date": datetime.now().isoformat(),
+            "version": item.get("version", "v0")
         }
-        
+
+        # Copy model metadata
+        for key in ["model_type", "base_model", "revision", "precision", "weight_type"]:
+            if key in item:
+                entry[key] = item[key]
+
         # Process per-category metrics
         if "per_category_metrics" in item:
             entry["per_category_metrics"] = item["per_category_metrics"]
-        
+
         # Process average metrics
         if "avg_metrics" in item:
             entry["avg_metrics"] = item["avg_metrics"]
-        
+
         entries.append(entry)
-    
+
     return entries
 
 
@@ -71,17 +94,23 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
     Convert leaderboard data to a pandas DataFrame for display.
     """
     rows = []
-    
+
     for entry in leaderboard_data.get("entries", []):
         model_name = entry.get("model_name", "Unknown Model")
-        
+
         # Extract average metrics for main display
         row = {
             "model_name": model_name,
             "model_type": entry.get("model_type", "Unknown"),
-            "submission_date": entry.get("submission_date", "")
+            "submission_date": entry.get("submission_date", ""),
+            "version": entry.get("version", "v0")
         }
-        
+
+        # Add additional metadata fields if present
+        for key in ["base_model", "revision", "precision", "weight_type"]:
+            if key in entry:
+                row[key] = entry[key]
+
         # Add average metrics
         avg_metrics = entry.get("avg_metrics", {})
         for test_type in TEST_TYPES:
@@ -90,12 +119,12 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
                     if metric in avg_metrics[test_type]:
                         col_name = f"{test_type}_{metric}"
                         row[col_name] = avg_metrics[test_type][metric]
-        
+
         # Calculate overall averages for key metrics
         f1_values = []
         recall_values = []
         precision_values = []
-        
+
         for test_type in TEST_TYPES:
             if test_type in avg_metrics and "f1_binary" in avg_metrics[test_type]:
                 f1_values.append(avg_metrics[test_type]["f1_binary"])
@@ -103,7 +132,7 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
                 recall_values.append(avg_metrics[test_type]["recall_binary"])
             if test_type in avg_metrics and "precision_binary" in avg_metrics[test_type]:
                 precision_values.append(avg_metrics[test_type]["precision_binary"])
-        
+
         # Add overall averages
         if f1_values:
             row["average_f1"] = sum(f1_values) / len(f1_values)
@@ -111,7 +140,7 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
             row["average_recall"] = sum(recall_values) / len(recall_values)
         if precision_values:
             row["average_precision"] = sum(precision_values) / len(precision_values)
-        
+
         # Add specific test type F1 scores for display
         if "default_prompts" in avg_metrics and "f1_binary" in avg_metrics["default_prompts"]:
             row["default_prompts_f1"] = avg_metrics["default_prompts"]["f1_binary"]
@@ -121,14 +150,14 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
             row["default_answers_f1"] = avg_metrics["default_answers"]["f1_binary"]
         if "jailbreaked_answers" in avg_metrics and "f1_binary" in avg_metrics["jailbreaked_answers"]:
             row["jailbreaked_answers_f1"] = avg_metrics["jailbreaked_answers"]["f1_binary"]
-        
+
         rows.append(row)
-    
+
     # Create DataFrame and sort by average F1 score
     df = pd.DataFrame(rows)
     if not df.empty and "average_f1" in df.columns:
         df = df.sort_values(by="average_f1", ascending=False)
-    
+
     return df
 
 
@@ -136,25 +165,29 @@ def add_entries_to_leaderboard(leaderboard_data: Dict, new_entries: List[Dict])
     """
     Add new entries to the leaderboard, replacing any with the same model name.
     """
-    # Create a mapping of existing entries by model name
-    existing_entries = {entry["model_name"]: i for i, entry in enumerate(leaderboard_data.get("entries", []))}
-    
+    # Create a mapping of existing entries by model name and version
+    existing_entries = {
+        (entry["model_name"], entry.get("version", "v0")): i
+        for i, entry in enumerate(leaderboard_data.get("entries", []))
+    }
+
     # Process each new entry
     for new_entry in new_entries:
         model_name = new_entry.get("model_name")
-        
-        if model_name in existing_entries:
+        version = new_entry.get("version", "v0")
+
+        if (model_name, version) in existing_entries:
             # Replace existing entry
-            leaderboard_data["entries"][existing_entries[model_name]] = new_entry
+            leaderboard_data["entries"][existing_entries[(model_name, version)]] = new_entry
         else:
             # Add new entry
             if "entries" not in leaderboard_data:
                 leaderboard_data["entries"] = []
             leaderboard_data["entries"].append(new_entry)
-    
+
     # Update the last_updated timestamp
     leaderboard_data["last_updated"] = datetime.now().isoformat()
-    
+
     return leaderboard_data
 
 
@@ -171,10 +204,10 @@ def process_jsonl_submission(file_path: str) -> Tuple[List[Dict], str]:
                     entries.append(entry)
                 except json.JSONDecodeError as e:
                     return [], f"Invalid JSON in submission file: {e}"
-        
+
         if not entries:
             return [], "Submission file is empty"
-        
+
         return entries, "Successfully processed submission"
     except Exception as e:
         return [], f"Error processing submission file: {e}"

src/populate.py

@@ -17,15 +17,29 @@ from src.envs import RESULTS_DATASET_ID, TOKEN, LEADERBOARD_FILE, CACHE_PATH
 from src.leaderboard.processor import leaderboard_to_dataframe, load_leaderboard_data, save_leaderboard_data, process_jsonl_submission, add_entries_to_leaderboard
 
 
-def download_leaderboard_data() -> bool:
+def get_versioned_leaderboard_file(version="v0"):
+    """
+    Get the versioned leaderboard file path.
+    """
+    base_name, ext = os.path.splitext(LEADERBOARD_FILE)
+    return f"{base_name}_{version}{ext}"
+
+
+def download_leaderboard_data(version="v0") -> bool:
     """
     Download the latest leaderboard data from HuggingFace.
+
+    Args:
+        version: The dataset version to download
     """
     try:
         # Create a temporary directory to download the submissions
-        temp_dir = os.path.join(CACHE_PATH, "temp_submissions")
+        temp_dir = os.path.join(CACHE_PATH, f"temp_submissions_{version}")
         os.makedirs(temp_dir, exist_ok=True)
 
+        # Get the versioned leaderboard file
+        leaderboard_file = get_versioned_leaderboard_file(version)
+
         # Download the entire repository
         try:
             snapshot_path = snapshot_download(
@@ -43,25 +57,43 @@ def download_leaderboard_data() -> bool:
 
             # Look for submission files in the submissions directory
             submissions_dir = os.path.join(snapshot_path, "submissions")
+            version_submissions_dir = os.path.join(snapshot_path, f"submissions_{version}")
+
+            # Check both standard and versioned submission directories
             if os.path.exists(submissions_dir):
                 submission_files.extend(glob(os.path.join(submissions_dir, "*.jsonl")))
 
-            # Also look for any JSONL files in the root
-            submission_files.extend(glob(os.path.join(snapshot_path, "*.jsonl")))
+            if os.path.exists(version_submissions_dir):
+                submission_files.extend(glob(os.path.join(version_submissions_dir, "*.jsonl")))
+
+            # Also look for any versioned JSONL files in the root
+            submission_files.extend(glob(os.path.join(snapshot_path, f"*_{version}.jsonl")))
+
+            # If we're looking for v0 and no versioned files found, use generic ones
+            if version == "v0" and not submission_files:
+                submission_files.extend(glob(os.path.join(snapshot_path, "*.jsonl")))
 
             # Process each submission file
             for file_path in submission_files:
                 entries, _ = process_jsonl_submission(file_path)
-                all_entries.extend(entries)
+
+                # Filter entries to those that match the version or don't have version specified
+                filtered_entries = [
+                    entry for entry in entries
+                    if entry.get("version", "v0") == version or "version" not in entry
+                ]
+
+                all_entries.extend(filtered_entries)
 
             # Create leaderboard data structure
             leaderboard_data = {
                 "entries": all_entries,
-                "last_updated": pd.Timestamp.now().isoformat()
+                "last_updated": pd.Timestamp.now().isoformat(),
+                "version": version
             }
 
             # Save to local file
-            save_leaderboard_data(leaderboard_data, LEADERBOARD_FILE)
+            save_leaderboard_data(leaderboard_data, leaderboard_file)
 
             return True
         except Exception as e:
@@ -72,7 +104,14 @@ def download_leaderboard_data() -> bool:
                 api = HfApi(token=TOKEN)
                 files = api.list_repo_files(repo_id=RESULTS_DATASET_ID, repo_type="dataset")
 
-                submission_files = [f for f in files if f.endswith('.jsonl')]
+                # Look for versioned and regular files
+                submission_files = [
+                    f for f in files
+                    if (f.endswith(f'_{version}.jsonl') or
+                        f.startswith(f'submissions_{version}/') or
+                        (version == "v0" and f.endswith('.jsonl')))
+                ]
+
                 all_entries = []
 
                 for file_path in submission_files:
@@ -84,49 +123,70 @@ def download_leaderboard_data() -> bool:
                             token=TOKEN
                         )
                         entries, _ = process_jsonl_submission(local_path)
-                        all_entries.extend(entries)
+
+                        # Filter entries to those that match the version or don't have version specified
+                        filtered_entries = [
+                            entry for entry in entries
+                            if entry.get("version", "v0") == version or "version" not in entry
+                        ]
+
+                        all_entries.extend(filtered_entries)
                     except Exception as file_error:
                         print(f"Error downloading file {file_path}: {file_error}")
 
                 # Create leaderboard data structure
                 leaderboard_data = {
                     "entries": all_entries,
-                    "last_updated": pd.Timestamp.now().isoformat()
+                    "last_updated": pd.Timestamp.now().isoformat(),
+                    "version": version
                 }
 
                 # Save to local file
-                save_leaderboard_data(leaderboard_data, LEADERBOARD_FILE)
+                save_leaderboard_data(leaderboard_data, leaderboard_file)
 
                 return True
             except Exception as list_error:
                 print(f"Error listing repository files: {list_error}")
 
             # If we can't download anything, create an empty leaderboard
-            if not os.path.exists(LEADERBOARD_FILE):
-                empty_data = {"entries": [], "last_updated": pd.Timestamp.now().isoformat()}
-                save_leaderboard_data(empty_data, LEADERBOARD_FILE)
+            if not os.path.exists(leaderboard_file):
+                empty_data = {
+                    "entries": [],
+                    "last_updated": pd.Timestamp.now().isoformat(),
+                    "version": version
+                }
+                save_leaderboard_data(empty_data, leaderboard_file)
 
             return False
     except Exception as e:
         print(f"Error downloading leaderboard data: {e}")
 
         # Ensure we have at least an empty leaderboard file
-        if not os.path.exists(LEADERBOARD_FILE):
-            empty_data = {"entries": [], "last_updated": pd.Timestamp.now().isoformat()}
-            save_leaderboard_data(empty_data, LEADERBOARD_FILE)
+        leaderboard_file = get_versioned_leaderboard_file(version)
+        if not os.path.exists(leaderboard_file):
+            empty_data = {
+                "entries": [],
+                "last_updated": pd.Timestamp.now().isoformat(),
+                "version": version
+            }
+            save_leaderboard_data(empty_data, leaderboard_file)
 
         return False
 
 
-def get_leaderboard_df() -> pd.DataFrame:
+def get_leaderboard_df(version="v0") -> pd.DataFrame:
     """
     Get the leaderboard data as a DataFrame.
+
+    Args:
+        version: The dataset version to retrieve
     """
     # Try to download the latest data
-    download_leaderboard_data()
+    download_leaderboard_data(version=version)
 
     # Load from local file
-    leaderboard_data = load_leaderboard_data(LEADERBOARD_FILE)
+    leaderboard_file = get_versioned_leaderboard_file(version)
+    leaderboard_data = load_leaderboard_data(leaderboard_file)
 
     # Convert to DataFrame
     df = leaderboard_to_dataframe(leaderboard_data)
@@ -134,18 +194,20 @@ def get_leaderboard_df() -> pd.DataFrame:
     return df
 
 
-def get_category_leaderboard_df(category: str) -> pd.DataFrame:
+def get_category_leaderboard_df(category: str, version="v0") -> pd.DataFrame:
     """
     Get the leaderboard data filtered by a specific category.
 
     Args:
         category: The category to filter by (e.g., "Criminal, Violent, and Terrorist Activity")
+        version: The dataset version to retrieve
 
     Returns:
         DataFrame with metrics for the specified category
     """
     # Load the leaderboard data
-    leaderboard_data = load_leaderboard_data(LEADERBOARD_FILE)
+    leaderboard_file = get_versioned_leaderboard_file(version)
+    leaderboard_data = load_leaderboard_data(leaderboard_file)
 
     # Filter entries to only include those with data for the specified category
     filtered_entries = []
@@ -158,6 +220,7 @@ def get_category_leaderboard_df(category: str) -> pd.DataFrame:
                 "model_name": entry.get("model_name", "Unknown Model"),
                 "model_type": entry.get("model_type", "Unknown"),
                 "submission_date": entry.get("submission_date", ""),
+                "version": entry.get("version", version),
             }
 
             # Extract metrics for this category
@@ -189,7 +252,8 @@ def get_category_leaderboard_df(category: str) -> pd.DataFrame:
     # Create a new leaderboard data structure with the filtered entries
     filtered_leaderboard = {
         "entries": filtered_entries,
-        "last_updated": leaderboard_data.get("last_updated", pd.Timestamp.now().isoformat())
+        "last_updated": leaderboard_data.get("last_updated", pd.Timestamp.now().isoformat()),
+        "version": version
     }
 
     # Convert to DataFrame
@@ -198,14 +262,21 @@ def get_category_leaderboard_df(category: str) -> pd.DataFrame:
     return df
 
 
-def get_detailed_model_data(model_name: str) -> Dict:
+def get_detailed_model_data(model_name: str, version="v0") -> Dict:
     """
     Get detailed data for a specific model.
+
+    Args:
+        model_name: The name of the model to get data for
+        version: The dataset version to retrieve
     """
-    leaderboard_data = load_leaderboard_data(LEADERBOARD_FILE)
+    leaderboard_file = get_versioned_leaderboard_file(version)
+    leaderboard_data = load_leaderboard_data(leaderboard_file)
 
     for entry in leaderboard_data.get("entries", []):
-        if entry.get("model_name") == model_name:
+        # Check both the model name and version
+        entry_version = entry.get("version", "v0")
+        if entry.get("model_name") == model_name and (entry_version == version or entry_version is None):
             return entry
 
     return {}

src/submission/submit.py

@@ -25,33 +25,40 @@ def validate_submission(file_path: str) -> Tuple[bool, str]:
         entries, message = process_jsonl_submission(file_path)
         if not entries:
             return False, message
-        
+
         # Additional validation could be added here
-        
+
         return True, "Submission is valid"
     except Exception as e:
         return False, f"Error validating submission: {e}"
 
 
-def submit_to_hub(file_path: str, metadata: Dict, dataset_id: str, token: str) -> Tuple[bool, str]:
+def submit_to_hub(file_path: str, metadata: Dict, dataset_id: str, token: str, version="v0") -> Tuple[bool, str]:
     """
     Submit results to a HuggingFace dataset repository as individual files.
+
+    Args:
+        file_path: Path to the submission file
+        metadata: Metadata to include with the submission
+        dataset_id: The dataset repository ID
+        token: HuggingFace API token
+        version: The version of the benchmark used (e.g., "v0", "v1")
     """
     try:
         # Process the submission file to validate
         entries, message = process_jsonl_submission(file_path)
         if not entries:
             return False, message
-        
+
         # Generate a unique submission ID
         model_name = metadata.get("model_name", "unknown")
         model_name_safe = model_name.replace("/", "_").replace(" ", "_")
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
         submission_id = f"{model_name_safe}_{timestamp}"
-        
+
         # Create an API instance
         api = HfApi(token=token)
-        
+
         # Create a temporary file with metadata added
         with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as temp_file:
             # Add metadata to each entry
@@ -59,47 +66,58 @@ def submit_to_hub(file_path: str, metadata: Dict, dataset_id: str, token: str) -
                 # If the entry already has a model_name, don't override it
                 if "model_name" not in entry:
                     entry["model_name"] = metadata.get("model_name")
-                
+
                 # Add other metadata if not present
                 for key, value in metadata.items():
                     if key != "model_name" and key not in entry:
                         entry[key] = value
-                
+
+                # Ensure version is set
+                entry["version"] = version
+
                 # Write to temp file
                 temp_file.write(json.dumps(entry) + "\n")
-            
+
             temp_path = temp_file.name
-        
-        # Upload the file directly to the repository
-        submission_path = f"submissions/{submission_id}.jsonl"
+
+        # Upload the file to the version-specific directory
+        submission_path = f"submissions_{version}/{submission_id}_{version}.jsonl" if version != "v0" else f"submissions/{submission_id}.jsonl"
         api.upload_file(
             path_or_fileobj=temp_path,
             path_in_repo=submission_path,
             repo_id=dataset_id,
             repo_type="dataset",
-            commit_message=f"Add submission for {model_name}"
+            commit_message=f"Add submission for {model_name} (version {version})"
         )
-        
+
         # Clean up the temporary file
         os.unlink(temp_path)
-        
-        return True, f"Successfully uploaded submission for {model_name} to {dataset_id}"
+
+        return True, f"Successfully uploaded submission for {model_name} to {dataset_id} (version {version})"
     except Exception as e:
         return False, f"Error submitting to dataset: {e}"
 
 
-def process_submission(file_path: str, metadata: Dict) -> str:
+def process_submission(file_path: str, metadata: Dict, version="v0") -> str:
     """
     Process a submission to the GuardBench leaderboard.
+
+    Args:
+        file_path: Path to the submission file
+        metadata: Metadata to include with the submission
+        version: The version of the benchmark used (e.g., "v0", "v1")
     """
     # Validate submission file
     is_valid, validation_message = validate_submission(file_path)
     if not is_valid:
         return styled_error(validation_message)
-    
+
+    # Add version to metadata
+    metadata["version"] = version
+
     # Submit to HuggingFace dataset repository
-    success, message = submit_to_hub(file_path, metadata, RESULTS_DATASET_ID, TOKEN)
+    success, message = submit_to_hub(file_path, metadata, RESULTS_DATASET_ID, TOKEN, version=version)
     if not success:
         return styled_error(message)
-    
+
     return styled_message(f"Submission successful! {message}")