Updated workflow APIs, code clean up and minor functions for hf pipeline support

app.py

@@ -1,4 +1,5 @@
 import json
+import sys
 
 import datasets
 import gradio as gr
@@ -6,17 +7,20 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 from loguru import logger
 
-import populate
-from about import LEADERBOARD_INTRODUCTION_TEXT, LEADERBOARD_TITLE
+from envs import LOG_LEVEL
+
+# Set the log level to INFO
+logger.remove()
+logger.add(sys.stdout, level=LOG_LEVEL, diagnose=False)
+
 from app_configs import DEFAULT_SELECTIONS, THEME
-from components.leaderboard import create_leaderboard_interface
+from components.hf_pipelines import create_hf_pipeline_submission_interface
 from components.quizbowl.bonus import BonusInterface
 from components.quizbowl.tossup import TossupInterface
 from components.typed_dicts import PipelineInterfaceDefaults, TossupInterfaceDefaults
 from display.css_html_js import fonts_header, js_head, leaderboard_css
 from display.custom_css import css_bonus, css_pipeline, css_tossup
 from display.guide import BUILDING_MARKDOWN, QUICKSTART_MARKDOWN
-from display.utils import AutoEvalColumn, fields
 
 # Constants
 from envs import (
@@ -40,9 +44,11 @@ from envs import (
 from hf_datasets_utils import download_dataset_snapshot
 from shared.workflows import factory
 from shared.workflows.configs import AVAILABLE_MODELS
+from shared.workflows.llms import llm_cache
 
 
 def restart_space():
+    llm_cache.sync_to_hf()
     API.restart_space(repo_id=REPO_ID)
 
 
@@ -112,7 +118,7 @@ if __name__ == "__main__":
         with gr.Row():
             with gr.Column(scale=5):
                 gr.Markdown(
-                    f"## 🤖 Welcome to QANTA 2025 Quizbowl Arena!     ⇒ 🏆 [Leaderboard]({LEADERBOARD_URL}) ⇐"
+                    f"## 🤖 Welcome to QANTA 2025 Quizbowl Arena!     👉 🏆 [Leaderboard]({LEADERBOARD_URL}) 👈"
                     "\n### 🎲 Create, play around, and submit your quizbowl agents."
                     f"<br>📋 [Register]({REGISTRATION_URL}) to participate in our [QANTA 2025 Human-AI Quizbowl Competition]({COMPETITION_URL}).",
                     elem_classes="welcome-text",
@@ -140,10 +146,8 @@ if __name__ == "__main__":
                     **DEFAULT_SELECTIONS["bonus"], init_workflow=factory.create_simple_qb_bonus_workflow()
                 )
                 bonus_interface = BonusInterface(demo, browser_state, bonus_ds, AVAILABLE_MODELS, defaults)
-            # with gr.Tab("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id="leaderboard"):
-            #     gr.Markdown("<a id='leaderboard' href='#leaderboard'>QANTA Leaderboard</a>")
-            #     gr.Markdown(LEADERBOARD_INTRODUCTION_TEXT)
-            #     create_leaderboard_interface(demo)
+            # with gr.Tab("🤗 HuggingFace Pipelines", elem_id="hf-pipeline-tab", id="hf-pipeline-tab"):
+            #     hf_pipeline_interface = create_hf_pipeline_submission_interface(demo)
             with gr.Tab("❓ Help", id="help"):
                 with gr.Row():
                     with gr.Column():
@@ -153,6 +157,8 @@ if __name__ == "__main__":
 
         # Event Listeners
 
+        # This is used to retrieve the pipeline state user was working on before login.
+        # makes things less annoying when progress is lost due to login.
         login_btn.click(
             fn=presave_pipeline_state,
             inputs=[

shared/workflows

@@ -1 +1 @@
-Subproject commit 6f9f3742d977eca6a385d6dbc5f28b5b16287cf1
+Subproject commit e5b9e225ca82372ee86f6d340b1523d4574bed3d

src/components/model_pipeline/model_pipeline.py

@@ -13,7 +13,6 @@ from components.model_step.model_step import ModelStepComponent
 from components.structs import ModelStepUIState, PipelineState, PipelineUIState
 from components.utils import make_state
 from shared.workflows.structs import ModelStep, Workflow
-from shared.workflows.validators import WorkflowValidationError, WorkflowValidator
 
 from .state_manager import get_output_panel_state
 
@@ -117,7 +116,6 @@ class PipelineInterface:
                 return step_interface
 
             is_multi_step = n_steps > 1
-            # logger.debug(f"Rendering step {position} of {n_steps}")
 
             # Add step controls below
             with gr.Row(elem_classes="step-controls", visible=is_multi_step):

src/components/model_pipeline/state_manager.py

@@ -215,7 +215,6 @@ class PipelineStateManager:
         """Update a workflow from a YAML string."""
         try:
             workflow = self.parse_yaml_workflow(yaml_str, strict=True)
-            logger.debug(f"Validator: {self.validator}")
             self.validator and self.validator(workflow)
             state = self.pipeline_state_cls.from_workflow(workflow)
             return state.model_dump(), not change_state, gr.update(visible=False)

src/components/quizbowl/bonus.py

@@ -15,6 +15,7 @@ from display.formatting import styled_error
 from shared.workflows import factory
 from shared.workflows.metrics import evaluate_prediction
 from shared.workflows.qb_agents import QuizBowlBonusAgent
+from shared.workflows.runners import run_and_eval_bonus_dataset, run_and_evaluate_bonus
 from submission import submit
 
 from . import populate, validation
@@ -28,10 +29,10 @@ def process_bonus_results(results: list[dict]) -> pd.DataFrame:
     return pd.DataFrame(
         [
             {
-                "Part": f"Part {r['part_number']}",
-                "Correct?": "✅" if r["score"] == 1 else "❌",
+                "Part": f"Part {r['number']}",
+                "Correct?": "✅" if r["correct"] == 1 else "❌",
                 "Confidence": r["confidence"],
-                "Prediction": r["answer"],
+                "Prediction": r["guess"],
                 "Explanation": r["explanation"],
             }
             for r in results
@@ -39,20 +40,20 @@ def process_bonus_results(results: list[dict]) -> pd.DataFrame:
     )
 
 
-def initialize_eval_interface(example: dict, model_outputs: list[dict], input_vars: list[str]):
+def initialize_eval_interface(example: dict, part_outputs: list[dict], input_vars: list[str]):
     """Initialize the interface with example text."""
     try:
         html_content = create_bonus_html(example["leadin"], example["parts"])
 
         # Create confidence plot data
-        plot_data = create_bonus_confidence_plot(example["parts"], model_outputs)
+        plot_data = create_bonus_confidence_plot(example["parts"], part_outputs)
 
         # Store state
-        state = {"parts": example["parts"], "outputs": model_outputs}
+        state = {"parts": example["parts"], "outputs": part_outputs}
 
         # Preparing step outputs for the model
         step_outputs = {}
-        for i, output in enumerate(model_outputs):
+        for i, output in enumerate(part_outputs):
             key = f"part {i + 1}"
             step_outputs[key] = {k: v for k, v in output["step_outputs"].items() if k not in input_vars}
             if output["logprob"] is not None:
@@ -60,8 +61,9 @@ def initialize_eval_interface(example: dict, model_outputs: list[dict], input_va
 
         return html_content, plot_data, state, step_outputs
     except Exception as e:
-        logger.exception(f"Error initializing interface: {e.args}")
-        return f"<div>Error initializing interface: {str(e)}</div>", pd.DataFrame(), {}, {}
+        error_msg = f"Error initializing interface: {str(e)}"
+        logger.exception(error_msg)
+        return styled_error(error_msg), pd.DataFrame(), {}, {}
 
 
 class BonusInterface:
@@ -79,19 +81,23 @@ class BonusInterface:
         self.render()
 
     # ------------------------------------- LOAD PIPELINE STATE FROM BROWSER STATE -------------------------------------
+    def load_default_workflow(self):
+        workflow = self.defaults["init_workflow"]
+        pipeline_state_dict = PipelineState.from_workflow(workflow).model_dump()
+        return pipeline_state_dict, {}
 
     def load_presaved_pipeline_state(self, browser_state: dict, pipeline_change: bool):
-        logger.debug(f"Loading presaved pipeline state from browser state:\n{json.dumps(browser_state, indent=4)}")
         try:
             state_dict = browser_state["bonus"].get("pipeline_state", {})
-            pipeline_state = PipelineState.model_validate(state_dict)
-            pipeline_state_dict = pipeline_state.model_dump()
-            output_state = browser_state["bonus"].get("output_state", {})
+            if state_dict:
+                pipeline_state = PipelineState.model_validate(state_dict)
+                pipeline_state_dict = pipeline_state.model_dump()
+                output_state = browser_state["bonus"].get("output_state", {})
+            else:
+                pipeline_state_dict, output_state = self.load_default_workflow()
         except Exception as e:
             logger.warning(f"Error loading presaved pipeline state: {e}")
-            output_state = {}
-            workflow = self.defaults["init_workflow"]
-            pipeline_state_dict = PipelineState.from_workflow(workflow).model_dump()
+            pipeline_state_dict, output_state = self.load_default_workflow()
         return browser_state, not pipeline_change, pipeline_state_dict, output_state
 
     # ------------------------------------------ INTERFACE RENDER FUNCTIONS -------------------------------------------
@@ -101,6 +107,7 @@ class BonusInterface:
             self.pipeline_selector = commons.get_pipeline_selector([])
             self.load_btn = gr.Button("⬇️ Import Pipeline", variant="secondary")
         self.import_error_display = gr.HTML(label="Import Error", elem_id="import-error-display", visible=False)
+        logger.info(f"Rendering {self.__class__.__name__} with pipeline state: {pipeline_state}")
         self.pipeline_interface = PipelineInterface(
             self.app,
             pipeline_state.workflow,
@@ -135,7 +142,7 @@ class BonusInterface:
     def render(self):
         """Create the Gradio interface."""
         self.hidden_input = gr.Textbox(value="", visible=False, elem_id="hidden-index")
-        workflow = factory.create_empty_tossup_workflow()
+        workflow = factory.create_empty_bonus_workflow()
         pipeline_state = PipelineState.from_workflow(workflow)
 
         with gr.Row():
@@ -195,25 +202,7 @@ class BonusInterface:
             error_msg = styled_error(f"Error loading pipeline: {str(e)}")
             return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
 
-    # ------------------------------------- Agent Functions -----------------------------------------------------------
-
-    def get_agent_outputs(self, example: dict, pipeline_state: PipelineState):
-        """Get the model outputs for a given question ID."""
-        outputs = []
-        leadin = example["leadin"]
-        agent = QuizBowlBonusAgent(pipeline_state.workflow)
-
-        for i, part in enumerate(example["parts"]):
-            # Run model for each part
-            part_output = agent.run(leadin, part["part"])
-
-            # Add part number and evaluate score
-            part_output["part_number"] = i + 1
-            part_output["score"] = evaluate_prediction(part_output["answer"], part["clean_answers"])
-
-            outputs.append(part_output)
-
-        return outputs
+        # ------------------------------------- Agent Functions -----------------------------------------------------------
 
     def single_run(
         self,
@@ -237,13 +226,14 @@ class BonusInterface:
                 raise gr.Error("Invalid question ID or dataset not loaded")
 
             example = self.ds[question_id]
-            outputs = self.get_agent_outputs(example, pipeline_state)
-
+            agent = QuizBowlBonusAgent(pipeline_state.workflow)
+            model_output = run_and_evaluate_bonus(agent, example, return_extras=True)
+            part_outputs = model_output["part_outputs"]
             # Process results and prepare visualization data
             html_content, plot_data, output_state, step_outputs = initialize_eval_interface(
-                example, outputs, pipeline_state.workflow.inputs
+                example, part_outputs, pipeline_state.workflow.inputs
             )
-            df = process_bonus_results(outputs)
+            df = process_bonus_results(part_outputs)
 
             return (
                 html_content,
@@ -254,7 +244,7 @@ class BonusInterface:
             )
         except Exception as e:
             error_msg = styled_error(create_error_message(e))
-            logger.exception(f"Error running tossup: {e}")
+            logger.exception(f"Error running bonus: {e}")
             return (
                 gr.skip(),
                 gr.skip(),
@@ -271,27 +261,26 @@ class BonusInterface:
             if not self.ds or not self.ds.num_rows:
                 return "No dataset loaded", None, None
 
-            total_correct = 0
+            agent = QuizBowlBonusAgent(pipeline_state.workflow)
+            model_outputs = run_and_eval_bonus_dataset(
+                agent, self.ds, num_workers=2, return_extras=True, tqdm_provider=progress.tqdm
+            )
+            n_parts_correct = 0
             total_parts = 0
-            part_scores = []
-            part_numbers = []
-
-            for example in progress.tqdm(self.ds, desc="Evaluating bonus questions"):
-                model_outputs = self.get_agent_outputs(example, pipeline_state)
-
-                for output in model_outputs:
-                    total_parts += 1
-                    if output["score"] == 1:
-                        total_correct += 1
-                    part_scores.append(output["score"])
-                    part_numbers.append(output["part_number"])
-
-            accuracy = total_correct / total_parts
+            n_questions_correct = 0
+            for model_output in model_outputs:
+                part_outputs = model_output["part_outputs"]
+                n_parts_correct += sum(output["correct"] for output in part_outputs)
+                total_parts += len(part_outputs)
+                n_questions_correct += int(n_parts_correct == len(part_outputs))
+
+            p_accuracy = n_parts_correct / total_parts
+            q_accuracy = n_questions_correct / len(self.ds)
             df = pd.DataFrame(
                 [
                     {
-                        "Part Accuracy": f"{accuracy:.2%}",
-                        "Total Score": f"{total_correct}/{total_parts}",
+                        "Question Accuracy": f"{q_accuracy:.2%}",
+                        "Part Accuracy": f"{p_accuracy:.2%}",
                         "Questions Evaluated": len(self.ds),
                     }
                 ]
@@ -305,7 +294,7 @@ class BonusInterface:
             )
         except Exception as e:
             error_msg = styled_error(create_error_message(e))
-            logger.exception(f"Error evaluating tossups: {e}")
+            logger.exception(f"Error evaluating bonus: {e}")
             return gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
 
     def submit_model(

src/components/quizbowl/plotting.py

@@ -37,14 +37,14 @@ def _create_token_tooltip_html(values) -> str:
         return ""
     confidence = values.get("confidence", 0)
     buzz = values.get("buzz", 0)
-    score = values.get("score", 0)
-    answer = values.get("answer", "")
-    answer_tokens = answer.split()
-    if len(answer_tokens) > 10:
-        k = len(answer_tokens) - 10
-        answer = " ".join(answer_tokens[:10]) + f"...[{k} more words]"
+    correct = values.get("correct", 0)
+    guess = values.get("guess", "")
+    guess_tokens = guess.split()
+    if len(guess_tokens) > 10:
+        k = len(guess_tokens) - 10
+        guess = " ".join(guess_tokens[:10]) + f"...[{k} more words]"
 
-    color = "#a3c9a3" if score else "#ebbec4"  # Light green for correct, light pink for incorrect
+    color = "#a3c9a3" if correct else "#ebbec4"  # Light green for correct, light pink for incorrect
 
     if values.get("logprob", None) is not None:
         prob = np.exp(values["logprob"])
@@ -56,10 +56,10 @@ def _create_token_tooltip_html(values) -> str:
         <div class="tooltip card" style="background-color: {color}; border-radius: 8px; padding: 12px; box-shadow: 2px 4px 8px rgba(0, 0, 0, 0.15);">
             <div class="tooltip-content" style="font-family: 'Arial', sans-serif; color: #000;">
                 <h4 style="margin: 0 0 8px; color: #000;">💡 Answer</h4>
-                <p><code style="font-weight: bold; margin: 0 0 8px; color: #000;">{answer}</code></p>
+                <p><code style="font-weight: bold; margin: 0 0 8px; color: #000;">{guess}</code></p>
                 <p style="margin: 0 0 4px; color: #000;">📈 <b style="color: #000;">Confidence:</b> {confidence:.2f}</p>
                 {prob_str}
-                <p style="margin: 0; color: #000;">🔍 <b style="color: #000;">Status:</b> {"✅ Correct" if score else "❌ Incorrect" if buzz else "🚫 No Buzz"}</p>
+                <p style="margin: 0; color: #000;">🔍 <b style="color: #000;">Status:</b> {"✅ Correct" if correct else "❌ Incorrect" if buzz else "🚫 No Buzz"}</p>
             </div>
         </div>
     """
@@ -68,14 +68,14 @@ def _create_token_tooltip_html(values) -> str:
 def create_token_html(token: str, values: dict, i: int) -> str:
     confidence = values.get("confidence", None)
     buzz = values.get("buzz", 0)
-    score = values.get("score", 0)
+    correct = values.get("correct", 0)
 
     # Replace non-word characters for proper display in HTML
     display_token = f"{token} 🚨" if buzz else f"{token} 💭" if values else token
     if not re.match(r"\w+", token):
         display_token = token.replace(" ", "&nbsp;")
 
-    css_class = _get_token_classes(confidence, buzz, score)
+    css_class = _get_token_classes(confidence, buzz, correct)
     # Add tooltip if we have values for this token
     tooltip_html = _create_token_tooltip_html(values)
 
@@ -98,8 +98,8 @@ def create_tossup_html(
         marker_indices = set(marker_indices)
 
         html_tokens = []
-        for i, token in enumerate(tokens):
-            token_html = create_token_html(token, ep.get(i, {}), i + 1)
+        for i, token in enumerate(tokens, start=1):
+            token_html = create_token_html(token, ep.get(i, {}), i)
             html_tokens.append(token_html)
 
         answer_html = _make_answer_html(answer_primary, clean_answers)
@@ -156,7 +156,7 @@ def create_bonus_html(leadin: str, parts: list[dict]) -> str:
 
 def create_tossup_confidence_pyplot(
     tokens: list[str],
-    eval_points: list[tuple[int, dict]],
+    run_outputs: list[dict],
     confidence_threshold: float = 0.5,
     prob_threshold: float | None = None,
 ) -> plt.Figure:
@@ -164,25 +164,26 @@ def create_tossup_confidence_pyplot(
     plt.style.use("ggplot")  # Set theme to grid paper
     fig = plt.figure(figsize=(10, 4), dpi=300)  # Set figure size to 11x5
     ax = fig.add_subplot(111)
-    x = [0] + [int(i + 1) for i, _ in eval_points]
-    y_conf = [0] + [v["confidence"] for _, v in eval_points]
-    logprob_values = [v["logprob"] for _, v in eval_points if v["logprob"] is not None]
-    y_prob = [0] + [np.exp(v) for v in logprob_values]
+    x = [0] + [o["token_position"] for o in run_outputs]
+    y_conf = [0] + [o["confidence"] for o in run_outputs]
+    logprobs = [o["logprob"] for o in run_outputs if o["logprob"] is not None]
+    y_prob = [0] + [np.exp(v) for v in logprobs]
 
     ax.plot(x, y_prob, "o-", color="#f2b150", label="Probability")
     ax.plot(x, y_conf, "o-", color="#4996de", label="Confidence")
-    for i, v in eval_points:
-        if not v["buzz"]:
+    for o in run_outputs:
+        if not o["buzz"]:
             continue
-        color = "green" if v["score"] else "red"
-        conf = v["confidence"]
-        ax.plot(i + 1, conf, "o", color=color, markerfacecolor="none", markersize=12, markeredgewidth=2.5)
-        if v["logprob"] is not None:
-            prob = np.exp(v["logprob"])
-            ax.plot(i + 1, prob, "o", color=color, markerfacecolor="none", markersize=12, markeredgewidth=2.5)
-        if i >= len(tokens):
-            print(f"Token index {i} is out of bounds for n_tokens: {len(tokens)}")
-        ax.annotate(f"{tokens[i]}", (i + 1, conf), textcoords="offset points", xytext=(0, 10), ha="center")
+        color = "green" if o["correct"] else "red"
+        conf = o["confidence"]
+        i = o["token_position"]
+        ax.plot(i, conf, "o", color=color, markerfacecolor="none", markersize=12, markeredgewidth=2.5)
+        if o["logprob"] is not None:
+            prob = np.exp(o["logprob"])
+            ax.plot(i, prob, "o", color=color, markerfacecolor="none", markersize=12, markeredgewidth=2.5)
+        if i > len(tokens):
+            print(f"1-indexed token index {i} is out of bounds for n_tokens: {len(tokens)}")
+        ax.annotate(f"{tokens[i - 1]}", (i, conf), textcoords="offset points", xytext=(0, 10), ha="center")
 
     # Add horizontal dashed line for confidence threshold
     ax.axhline(y=confidence_threshold, color="#9370DB", linestyle="--", xmin=0, xmax=1, label="Confidence Threshold")
@@ -228,7 +229,7 @@ def create_bonus_confidence_plot(parts: list[dict], model_outputs: list[dict]) -
     # Plot confidence for each part
     x = range(1, len(parts) + 1)
     confidences = [output["confidence"] for output in model_outputs]
-    scores = [output["score"] for output in model_outputs]
+    scores = [output["correct"] for output in model_outputs]
 
     # Plot confidence bars
     bars = ax.bar(x, confidences, color="#4698cf")
@@ -287,13 +288,16 @@ def create_tossup_eval_table(df: pd.DataFrame) -> pd.DataFrame:
     pos_gaps = gaps.loc[gaps >= 0]
     neg_gaps = gaps.loc[gaps < 0]
 
-    mean_tossup_score = df["tossup_score"].sum() / len(df)
+    mean_tossup_score = df["raw_score"].sum() / len(df)
+    expected_score = df["expected_score"].sum() / len(df)
+    buzz_precision = df["is_correct"].sum() / df["buzz"].sum()
 
     return pd.DataFrame(
         [
             {
-                "Tossup Score (10)": f"{mean_tossup_score:5.1f}",
-                "Buzz Accuracy": f"{df['is_correct'].mean():5.1%}",
+                "Raw Score": f"{mean_tossup_score:5.1f}",
+                "Expected Score": f"{expected_score:5.1f}",
+                "Buzz Precision": f"{buzz_precision:5.1%}",
                 "Buzz Position": f"{np.mean(positions):5.1f}",
                 "+ve Gap": f"{pos_gaps.mean():5.1f}",
                 "-ve Gap": f"{neg_gaps.mean():5.1f}",
@@ -493,16 +497,16 @@ def create_dummy_model_outputs(n_entries=10, n_positions=5):
 
             outputs.append(
                 {
-                    "position": i + 1,
+                    "run_idx": i + 1,
                     "buzz": will_buzz,
-                    "score": 1 if is_correct else 0,
+                    "correct": 1 if is_correct else 0,
                     "confidence": np.random.random(),
                     "logprob": np.log(np.random.random()),
-                    "answer": f"Answer {i + 1}",
+                    "guess": f"Answer {i + 1}",
                 }
             )
 
-        dummy_outputs.append({"run_indices": run_indices, "outputs": outputs})
+        dummy_outputs.append({"run_indices": run_indices, "run_outputs": outputs})
 
     return dummy_outputs
 

src/components/quizbowl/tossup.py

@@ -16,6 +16,7 @@ from shared.workflows import factory
 from shared.workflows.metrics import evaluate_prediction
 from shared.workflows.metrics.qb_metrics import prepare_tossup_results_df
 from shared.workflows.qb_agents import QuizBowlTossupAgent, TossupResult
+from shared.workflows.runners import run_and_eval_tossup_dataset, run_and_evaluate_tossup
 from submission import submit
 
 from . import populate, validation
@@ -28,9 +29,6 @@ from .plotting import (
 from .utils import create_error_message
 from .validation import UserInputWorkflowValidator
 
-# TODO: Error handling on run tossup and evaluate tossup and show correct messages
-# TODO: ^^ Same for Bonus
-
 
 class ScoredTossupResult(TossupResult):
     """Result of a tossup question with evaluation score and position."""
@@ -44,8 +42,8 @@ def add_model_scores(
 ) -> list[ScoredTossupResult]:
     """Add model scores to the model outputs."""
     for output in run_outputs:
-        output["score"] = evaluate_prediction(output["answer"], clean_answers)
-        output["token_position"] = run_indices[output["position"] - 1]
+        output["correct"] = evaluate_prediction(output["guess"], clean_answers)
+        output["token_position"] = run_indices[output["run_idx"] - 1]
     return run_outputs
 
 
@@ -58,7 +56,7 @@ def prepare_buzz_evals(
         return [], []
     eval_points = []
     for o in model_outputs:
-        token_position = run_indices[o["position"] - 1]
+        token_position = run_indices[o["run_idx"] - 1]
         eval_points.append((token_position, o))
 
     return eval_points
@@ -80,9 +78,11 @@ def initialize_eval_interface(
         eval_points = [(o["token_position"], o) for o in run_outputs]
 
         if not tokens:
-            return "<div>No tokens found in the provided text.</div>", pd.DataFrame(), "{}"
+            error_msg = "No tokens found in the provided text."
+            logger.exception(error_msg)
+            return styled_error(error_msg), pd.DataFrame(), {}, {}
         html_content = create_tossup_html(tokens, answer, clean_answers, run_indices, eval_points)
-        plot_data = create_tossup_confidence_pyplot(tokens, eval_points, confidence_threshold, prob_threshold)
+        plot_data = create_tossup_confidence_pyplot(tokens, run_outputs, confidence_threshold, prob_threshold)
 
         # Store tokens, values, and buzzes as JSON for later use
         state = {"tokens": tokens, "values": eval_points}
@@ -91,15 +91,16 @@ def initialize_eval_interface(
         step_outputs = {}
         for output in run_outputs:
             tok_pos = output["token_position"]
-            key = "{pos}:{token}".format(pos=tok_pos + 1, token=tokens[tok_pos])
+            key = "{pos}:{token}".format(pos=tok_pos, token=tokens[tok_pos - 1])
             step_outputs[key] = {k: v for k, v in output["step_outputs"].items() if k not in input_vars}
             if output["logprob"] is not None:
                 step_outputs[key]["output_probability"] = float(np.exp(output["logprob"]))
 
         return html_content, plot_data, state, step_outputs
     except Exception as e:
-        logger.exception(f"Error initializing interface: {e.args}")
-        return f"<div>Error initializing interface: {str(e)}</div>", pd.DataFrame(), "{}", {}
+        error_msg = f"Error initializing interface: {str(e)}"
+        logger.exception(error_msg)
+        return styled_error(error_msg), pd.DataFrame(), {}, {}
 
 
 def process_tossup_results(results: list[dict]) -> pd.DataFrame:
@@ -108,12 +109,12 @@ def process_tossup_results(results: list[dict]) -> pd.DataFrame:
     for r in results:
         entry = {
             "Token Position": r["token_position"],
-            "Correct?": "✅" if r["score"] == 1 else "❌",
+            "Correct?": "✅" if r["correct"] == 1 else "❌",
             "Confidence": r["confidence"],
         }
         if r["logprob"] is not None:
             entry["Probability"] = f"{np.exp(r['logprob']):.3f}"
-        entry["Prediction"] = r["answer"]
+        entry["Prediction"] = r["guess"]
         data.append(entry)
     return pd.DataFrame(data)
 
@@ -141,18 +142,23 @@ class TossupInterface:
 
     # ------------------------------------- LOAD PIPELINE STATE FROM BROWSER STATE ------------------------------------
 
+    def load_default_workflow(self):
+        workflow = self.defaults["init_workflow"]
+        pipeline_state_dict = TossupPipelineState.from_workflow(workflow).model_dump()
+        return pipeline_state_dict, {}
+
     def load_presaved_pipeline_state(self, browser_state: dict, pipeline_change: bool):
-        logger.debug(f"Loading presaved pipeline state from browser state:\n{json.dumps(browser_state, indent=4)}")
         try:
             state_dict = browser_state["tossup"].get("pipeline_state", {})
-            pipeline_state = TossupPipelineState.model_validate(state_dict)
-            pipeline_state_dict = pipeline_state.model_dump()
-            output_state = browser_state["tossup"].get("output_state", {})
+            if state_dict:
+                pipeline_state = TossupPipelineState.model_validate(state_dict)
+                pipeline_state_dict = pipeline_state.model_dump()
+                output_state = browser_state["tossup"].get("output_state", {})
+            else:
+                pipeline_state_dict, output_state = self.load_default_workflow()
         except Exception as e:
             logger.warning(f"Error loading presaved pipeline state: {e}")
-            output_state = {}
-            workflow = self.defaults["init_workflow"]
-            pipeline_state_dict = TossupPipelineState.from_workflow(workflow).model_dump()
+            pipeline_state_dict, output_state = self.load_default_workflow()
         return browser_state, not pipeline_change, pipeline_state_dict, output_state
 
     # ------------------------------------------ INTERFACE RENDER FUNCTIONS -------------------------------------------
@@ -256,18 +262,6 @@ class TossupInterface:
             return UNSELECTED_PIPELINE_NAME, gr.skip(), gr.skip(), gr.update(visible=True, value=error_msg)
 
     # ------------------------------------- Agent Functions -----------------------------------------------------------
-    def get_agent_outputs(
-        self, example: dict, pipeline_state: TossupPipelineState, early_stop: bool
-    ) -> list[ScoredTossupResult]:
-        """Get the model outputs for a given question ID."""
-        question_runs = []
-        tokens = example["question"].split()
-        for run_idx in example["run_indices"]:
-            question_runs.append(" ".join(tokens[: run_idx + 1]))
-        agent = QuizBowlTossupAgent(pipeline_state.workflow)
-        outputs = list(agent.run(question_runs, early_stop=early_stop))
-        outputs = add_model_scores(outputs, example["clean_answers"], example["run_indices"])
-        return outputs
 
     def single_run(
         self,
@@ -295,15 +289,20 @@ class TossupInterface:
             if not self.ds or question_id < 0 or question_id >= len(self.ds):
                 raise gr.Error("Invalid question ID or dataset not loaded")
             example = self.ds[question_id]
-            outputs = self.get_agent_outputs(example, pipeline_state, early_stop)
-
+            outputs = run_and_evaluate_tossup(
+                QuizBowlTossupAgent(pipeline_state.workflow),
+                example,
+                return_extras=True,
+                early_stop=early_stop,
+            )
+            run_outputs = outputs["run_outputs"]
             # Process results and prepare visualization data
             confidence_threshold = workflow.buzzer.confidence_threshold
             prob_threshold = workflow.buzzer.prob_threshold
             tokens_html, plot_data, output_state, step_outputs = initialize_eval_interface(
-                example, outputs, workflow.inputs, confidence_threshold, prob_threshold
+                example, run_outputs, workflow.inputs, confidence_threshold, prob_threshold
             )
-            df = process_tossup_results(outputs)
+            df = process_tossup_results(run_outputs)
 
             return (
                 tokens_html,
@@ -332,10 +331,10 @@ class TossupInterface:
             if not self.ds or not self.ds.num_rows:
                 return "No dataset loaded", None, None
             pipeline_state = validation.validate_tossup_workflow(state_dict)
-            model_outputs = []
-            for example in progress.tqdm(self.ds, desc="Evaluating tossup questions"):
-                run_outputs = self.get_agent_outputs(example, pipeline_state, early_stop=True)
-                model_outputs.append(run_outputs)
+            agent = QuizBowlTossupAgent(pipeline_state.workflow)
+            model_outputs = run_and_eval_tossup_dataset(
+                agent, self.ds, return_extras=True, tqdm_provider=progress.tqdm, num_workers=2
+            )
             eval_df = prepare_tossup_results_df(model_outputs, self.ds["run_indices"])
             plot_data = create_tossup_eval_dashboard(self.ds["run_indices"], eval_df)
             output_df = create_tossup_eval_table(eval_df)

src/components/quizbowl/utils.py

@@ -1,7 +1,3 @@
-from typing import Any, Dict, List
-
-import pandas as pd
-
 from shared.workflows.errors import ProviderAPIError, WorkflowExecutionError
 
 
@@ -15,61 +11,3 @@ def create_error_message(e: Exception) -> str:
         return f"Invalid input -- {e}. Please try again. \n\nIf the problem persists, please contact support."
     else:
         return "An unexpected error occurred. Please contact support."
-
-
-def _create_confidence_plot_data(results: List[Dict], top_k_mode: bool = False) -> pd.DataFrame:
-    """Create a DataFrame for the confidence plot."""
-    if not top_k_mode:
-        return pd.DataFrame(
-            {
-                "position": [r["position"] for r in results],
-                "confidence": [r["confidence"] for r in results],
-                "answer": [r["answer"] for r in results],
-            }
-        )
-
-    # For top-k mode, extract and plot top answers
-    return _create_top_k_plot_data(results)
-
-
-def _create_top_k_plot_data(results: List[Dict]) -> pd.DataFrame:
-    """Create plot data for top-k mode."""
-    # Find top answers across all positions (limited to top 5)
-    top_answers = set()
-    for r in results:
-        for g in r.get("guesses", [])[:3]:  # Get top 3 from each position
-            if g.get("answer"):
-                top_answers.add(g.get("answer"))
-
-    top_answers = list(top_answers)[:5]  # Limit to 5 total answers
-
-    # Create plot data for each answer
-    all_data = []
-    for position_idx, result in enumerate(results):
-        position = result["position"]
-        for answer in top_answers:
-            confidence = 0
-            for guess in result.get("guesses", []):
-                if guess.get("answer") == answer:
-                    confidence = guess.get("confidence", 0)
-                    break
-            all_data.append({"position": position, "confidence": confidence, "answer": answer})
-
-    return pd.DataFrame(all_data)
-
-
-def _create_top_k_dataframe(results: List[Dict]) -> pd.DataFrame:
-    """Create a DataFrame for top-k results."""
-    df_rows = []
-    for result in results:
-        position = result["position"]
-        for i, guess in enumerate(result.get("guesses", [])):
-            df_rows.append(
-                {
-                    "position": position,
-                    "answer": guess.get("answer", ""),
-                    "confidence": guess.get("confidence", 0),
-                    "rank": i + 1,
-                }
-            )
-    return pd.DataFrame(df_rows)

src/display/custom_css.py

@@ -12,6 +12,7 @@ css_pipeline = """
     --input-text-size: var(--text-sm) !important;
     --body-text-size: 14px !important;
     --input-background-fill-focus: var(--secondary-300) !important;
+    --link-text-color: blue !important;
 
     // Button Colors
     --button-primary-background-fill: var(--primary-800) !important;
@@ -38,6 +39,7 @@ css_pipeline = """
     --text-lg: 16px !important;
     --input-text-size: var(--text-sm) !important;
     --body-text-size: 14px !important;
+    --link-text-color: blue !important;
 
     --button-primary-background-fill: var(--neutral-100) !important;
     --button-secondary-background-fill: var(--secondary-300) !important;

src/envs.py

@@ -41,6 +41,7 @@ PLAYGROUND_DATASET_NAMES = {
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH = os.getenv("HF_HOME", ".")
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
 
 # Local caches
 LLM_CACHE_PATH = os.path.join(CACHE_PATH, "llm-cache")

src/submission/structs.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel, Field
 from shared.workflows.structs import TossupWorkflow, Workflow
 
 CompetitionType = Literal["tossup", "bonus"]
-SubmissionType = Literal["python_file", "simple_workflow", "complex_workflow"]
+SubmissionType = Literal["python_file", "simple_workflow", "complex_workflow", "hf_pipeline"]
 SubmissionStatus = Literal["submitted", "in_progress", "completed", "failed"]
 
 

src/submission/submit.py

@@ -40,7 +40,7 @@ def get_user_submissions(username: str, competition_type: str, pattern: str = No
 def get_user_submission_names(competition_type: str, profile: gr.OAuthProfile | None) -> list[str]:
     """Get all submission model names for a user."""
     if profile is None:
-        logger.warning("No user profile provided. Returning empty list.")
+        logger.info("No user profile provided. Returning empty list.")
         return []
     submissions = get_user_submissions(profile.username, competition_type)
     return [f"{s.username}/{s.model_name}" for s in submissions]
@@ -88,7 +88,7 @@ def get_time_until_next_submission(tz: timezone = timezone.utc) -> str:
     return remaining_time_str
 
 
-def create_submission(
+def create_workflow_submission(
     username: str,
     model_name: str,
     description: str,
@@ -125,6 +125,41 @@ def create_submission(
     return submission
 
 
+def create_hf_submission(
+    username: str,
+    model_name: str,
+    description: str,
+    competition_type: CompetitionType,
+) -> Submission:
+    """
+    Create a submission for a tossup model.
+
+    Args:
+        username: Username of the user who created the submission
+        model_name: Name of the model
+        description: Detailed description of what the submission does
+        competition_type: Type of competition
+
+    Returns:
+        Submission object if successful, None if validation fails
+    """
+    # Create the submission
+    dt = datetime.now(timezone.utc)
+    submission = Submission(
+        id=f"{competition_type}__hf__{dt.strftime('%Y%m%d_%H%M%S')}__{username}__{model_name.lower().replace(' ', '_')}",
+        model_name=model_name,
+        username=username,
+        description=description,
+        competition_type=competition_type,
+        submission_type="hf_pipeline",
+        status="submitted",
+        created_at=dt.isoformat(),
+        updated_at=dt.isoformat(),
+    )
+
+    return submission
+
+
 def validate_model_name(model_name: str):
     # check if model_name has no white spaces, no special characters apart from _ and -
     if " " in model_name:
@@ -177,7 +212,7 @@ def submit_model(
         return styled_error(f"Submission Error! Invalid model name '{model_name}'.<br>{error_msg}")
 
     try:
-        submission = create_submission(
+        submission = create_workflow_submission(
             username=username,
             model_name=model_name,
             description=description,