app.py

import gradio as gr
import pandas as pd
import os, json, datetime
from apscheduler.schedulers.background import BackgroundScheduler
from sklearn.metrics import mean_absolute_error
from huggingface_hub import login, hf_hub_download
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import base64

# Log in programmatically using token from secrets.
login(token=os.getenv("HF_TOKEN"))

# ---------- Load Image for Description ----------
with open("assets/image.png", "rb") as f:
    image_data = f.read()
encoded_image = base64.b64encode(image_data).decode("utf-8")


# ---------- CSV to COCO Conversion Functions ----------

def convert_gt_csv_to_coco(gt_df):
    """
    Convert ground truth DataFrame to COCO-style dictionary.
    Expected columns in gt_df: image_name, class_id, xmin, ymin, xmax, ymax, weight.
    """
    images = []
    annotations = []
    # Create a mapping for image names to unique IDs.
    image_names = gt_df["image_name"].unique()
    image_id_map = {name: idx + 1 for idx, name in enumerate(image_names)}
    
    # Create a list of categories from the unique class_ids.
    category_ids = sorted(gt_df["class_id"].unique())
    categories = [{"id": int(cat), "name": f"class_{int(cat)}"} for cat in category_ids]
    
    ann_id = 1
    for idx, row in gt_df.iterrows():
        img_name = row["image_name"]
        image_id = image_id_map[img_name]
        # Add image information if not already added.
        if image_id not in [img["id"] for img in images]:
            images.append({"id": image_id, "file_name": img_name})
        x = row["xmin"]
        y = row["ymin"]
        width = row["xmax"] - row["xmin"]
        height = row["ymax"] - row["ymin"]
        ann = {
            "id": ann_id,
            "image_id": image_id,
            "category_id": int(row["class_id"]),
            "bbox": [x, y, width, height],
            "area": width * height,
            "iscrowd": 0
        }
        annotations.append(ann)
        ann_id += 1
    coco_dict = {
        "images": images,
        "annotations": annotations,
        "categories": categories
    }
    return coco_dict, image_id_map

def convert_pred_csv_to_coco(pred_df, image_id_map):
    """
    Convert predictions DataFrame to COCO-style list of predictions.
    Expected columns in pred_df: image_name, class_id, xmin, ymin, xmax, ymax, weight, conf.
    """
    preds = []
    for idx, row in pred_df.iterrows():
        img_name = row["image_name"]
        image_id = image_id_map.get(img_name)
        if image_id is None:
            continue  # Skip predictions for images not in the ground truth.
        x = row["xmin"]
        y = row["ymin"]
        width = row["xmax"] - row["xmin"]
        height = row["ymax"] - row["ymin"]
        pred = {
            "image_id": image_id,
            "category_id": int(row["class_id"]),
            "bbox": [x, y, width, height],
            "score": row["conf"]
        }
        preds.append(pred)
    return preds

def compute_map50(gt_df, pred_df):
    """
    Convert CSV DataFrames to COCO format and compute mAP50 using pycocotools.
    Returns the mAP50 value.
    """
    coco_gt_dict, image_id_map = convert_gt_csv_to_coco(gt_df)
    coco_preds = convert_pred_csv_to_coco(pred_df, image_id_map)
    
    # Save temporary JSON files.
    gt_json_path = "temp_gt_coco.json"
    pred_json_path = "temp_pred_coco.json"
    with open(gt_json_path, "w") as f:
        json.dump(coco_gt_dict, f)
    with open(pred_json_path, "w") as f:
        json.dump(coco_preds, f)
        
    # Load ground truth and predictions with the COCO API.
    cocoGt = COCO(gt_json_path)
    try:
        cocoDt = cocoGt.loadRes(pred_json_path)
    except Exception as e:
        print("Error loading predictions into COCO:", e)
        return None

    cocoEval = COCOeval(cocoGt, cocoDt, iouType="bbox")
    # Set evaluation to only consider an IoU threshold of 0.50.
    cocoEval.params.iouThrs = [0.5]
    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()
    # stats[0] is typically the [email protected].
    return cocoEval.stats[0]


# ---------- Evaluation Functions (Combined Metrics) ----------

# Update leaderboard columns (here we remove explicit user and submission date fields).
LEADERBOARD_FILE = "evaluation_results.csv"
LEADERBOARD_COLUMNS = [
    "Name", "Model", "mAP@50", "weight_MAE (g)",
    "Paper", "Source Code", "Model Type", "Hub License"
]

def save_submission_result(result):
    if os.path.exists(LEADERBOARD_FILE):
        df = pd.read_csv(LEADERBOARD_FILE)
        df = pd.concat([df, pd.DataFrame([result])], ignore_index=True)
    else:
        df = pd.DataFrame([result], columns=LEADERBOARD_COLUMNS)
    df.to_csv(LEADERBOARD_FILE, index=False)

def load_leaderboard():
    if os.path.exists(LEADERBOARD_FILE):
        df = pd.read_csv(LEADERBOARD_FILE)
        # Sort by mAP@50 (higher is better)
        df = df.sort_values(by="mAP@50", ascending=False)
        return df
    else:
        return pd.DataFrame(columns=LEADERBOARD_COLUMNS)

# --- Rate Limiting by IP ---
# A global dictionary that will map client IPs to the timestamp of their last submission.
submissions_by_ip = {}
# Set the allowed interval between submissions in seconds. Here, 86400 seconds = 1 day.
SUBMISSION_INTERVAL_SECONDS = 86400

def evaluate_submission(file, submission_name, model_description, link_paper, link_code, model_type, hub_license, req: gr.Request):
    """
    Evaluate the submitted CSV by computing:
    - mAP@50 (using COCO evaluation) for bounding boxes.
    - Weight MAE for the food weight predictions.
    Also implements a simple rate limiting per IP address to help avoid spam.
    """
    # Get the client IP from the Gradio request.
    client_ip = req.client.host if req.client and req.client.host else "unknown"
    now = datetime.datetime.now()
    
    # Check if this IP has submitted recently.
    if client_ip in submissions_by_ip:
        last_sub_time = submissions_by_ip[client_ip]
        delta = (now - last_sub_time).total_seconds()
        if delta < SUBMISSION_INTERVAL_SECONDS:
            return f"❌ Only one submission per day is allowed. Please try again later."
    
    # Validate required fields.
    if not submission_name.strip():
        return "❌ Name field is required."
    if not model_description.strip():
        return "❌ Model field is required."
    if file is None:
        return "❌ Please upload a file."
    
    print("File received:", file)
    try:
        pred_df = pd.read_csv(file.name)
        print("Prediction CSV columns:", pred_df.columns.tolist())
    except Exception as e:
        return f"❌ Prediction CSV read failed: {e}"
    
    try:
        gt_path = hf_hub_download(
            repo_id="issai/ground-truth-food-eval",
            filename="ground_truth.csv",
            repo_type="dataset",
            token=True
        )
        gt_df = pd.read_csv(gt_path)
    except Exception as e:
        return "Error loading ground truth file: " + str(e)
    
    print("Ground truth shape:", gt_df.shape)
    print("Ground truth sample:")
    print(gt_df.head())
    print("Predictions shape:", pred_df.shape)
    print("Predictions sample:")
    print(pred_df.head())
    
    # --- Compute mAP50 ---
    ap50 = compute_map50(gt_df, pred_df)
    if ap50 is None:
        return "Error computing mAP@50."
    print("Computed mAP50:", ap50)
    
    # --- Compute Weight MAE ---
    try:
        df = pd.merge(gt_df, pred_df, on=["image_name", "class_id"], suffixes=("_gt", "_pred"))
    except Exception as e:
        return f"❌ Error during merge for MAE: {e}"
    
    df["weight_gt"] = pd.to_numeric(df["weight_gt"], errors="coerce")
    df["weight_pred"] = pd.to_numeric(df["weight_pred"], errors="coerce")
    
    before_filter = df.shape[0]
    df = df[(df["weight_gt"] != -1) & (df["weight_pred"] != -1)]
    filtered_out = before_filter - df.shape[0]
    print(f"Filtered out {filtered_out} rows for MAE due to weight == -1")
    
    if df.empty:
        weight_mae = 0.0
    else:
        weight_mae = mean_absolute_error(df["weight_gt"], df["weight_pred"])
    print("Computed Weight MAE:", weight_mae)
    
    # --- Save Submission Result ---
    result = {
        "Name": submission_name,
        "Model": model_description,
        "mAP@50": f"{ap50:.3f}",
        "weight_MAE (g)": f"{weight_mae:.3f}",
        "Paper": link_paper,
        "Source Code": link_code,
        "Model Type": model_type,
        "Hub License": hub_license,
    }
    
    save_submission_result(result)
    
    # Update the last submission time for this IP.
    submissions_by_ip[client_ip] = now
    
    result_text = (
        f"**Evaluation Results for '{submission_name}'**\n\n"
        f"- Model: {model_description}\n"
        f"- mAP@50: {ap50:.3f}\n"
        f"- Weight MAE: {weight_mae:.3f} grams\n"
        f"- Link to Paper: {link_paper}\n"
        f"- Link to Source Code: {link_code}\n"
        f"- Model Type: {model_type}\n"
        f"- Hub License: {hub_license}\n"
    )
    return result_text

def evaluate_and_refresh(file, submission_name, model_description, link_paper, link_code, model_type, hub_license, req: gr.Request):
    result_text = evaluate_submission(file, submission_name, model_description, link_paper, link_code, model_type, hub_license, req)
    updated_leaderboard = load_leaderboard()
    return result_text, updated_leaderboard


# ---------- Gradio Interface ----------

with gr.Blocks() as demo:
    gr.Markdown("# Benchmark Leaderboard: Food Object Detection & Food Weight Estimation")
    
    gr.HTML(f'<img src="data:image/png;base64,{encoded_image}" alt="Description" style="max-width:100%;height:auto;">')

    gr.Markdown(
        "This leaderboard evaluates submissions for the [Food Portion Benchmark dataset](https://huggingface.co/datasets/issai/Food_Portion_Benchmark) using **mAP@50** for bounding boxes "
        "and **Mean Absolute Error (MAE)** for food weight predictions. \n\n"
        "Submissions are expected in CSV format with the columns: `image_name, class_id, xmin, ymin, xmax, ymax, weight, conf`.\n\n"
        "The ground truth CSV (kept privately) has the columns: `image_name, class_id, xmin, ymin, xmax, ymax, weight`.\n\n"
    )

    gr.Markdown(
        """
        ### Sample Submission Template

        Download a sample CSV file to see the required format:
        [Download Sample Submission CSV](https://huggingface.co/spaces/issai/Food-Weight-Benchmark/resolve/main/sample_submission.csv)
        """
    )

    # --- Leaderboard Tab (Public) ---
    with gr.TabItem("🏅 Leaderboard"):
        leaderboard_output = gr.Dataframe(label="Leaderboard")
        demo.load(load_leaderboard, outputs=leaderboard_output)
        refresh_button = gr.Button("Refresh Leaderboard")
        refresh_button.click(fn=load_leaderboard, outputs=leaderboard_output)

    with gr.TabItem("📈 Metrics"):
        gr.Markdown(
            """
            ### Evaluation Metrics

            - **mAP@50 (Mean Average Precision at IoU 0.50):**  
            This metric evaluates how well the predicted bounding boxes match the ground truth. In mAP@50, a prediction is considered a true positive if the Intersection over Union (IoU) between the predicted box and the ground truth box is at least 0.50. The final score is averaged across all classes and images, yielding a single value between 0 and 1, where a higher value indicates better localization performance.

            - **Weight MAE (Mean Absolute Error):**  
            This metric calculates the average absolute difference (in grams) between the predicted food weight and the actual weight provided in the ground truth. A lower MAE signifies more accurate weight predictions.

            ### Benchmark Dataset

            The **Food Portion Benchmark dataset** is a comprehensive dataset for evaluating object detection and food weight estimation models. Here are some key details:

            - **Dataset Composition:**  
            It contains 14,083 RGB images of food items spanning 133 distinct classes. For each food item, the dataset includes manually annotated bounding boxes and precise weight measurements.

            - **Portion Sizes:**  
            Each food item is represented with annotations for three different portion sizes (big, average, small), reflecting the real-world variation in food serving sizes.

            - **Annotations:**  
            The ground truth annotations include the food item’s image name, class, bounding box coordinates in YOLO format, and weight in grams. 

            - **Reference and Access:**  
            You can explore and download the dataset on Hugging Face at the following link:  
            [Food Portion Benchmark Dataset](https://huggingface.co/datasets/issai/Food_Portion_Benchmark)

            ### Additional Notes

            - **Submission Requirements:**  
            Prediction CSV files must contain: image_name, class_id, xmin, ymin, xmax, ymax, weight, conf  
        
            - **Evaluation Process:**  
                - **mAP@50** is computed via the COCO evaluation API using pycocotools library, which compares the predicted bounding boxes (along with their confidence scores) to the ground truth annotations.
                - **Weight MAE** is computed using sklearn’s mean absolute error function.

            - **Contact Information:**  
            For any questions regarding the dataset or evaluation methodology, please refer to the dataset documentation on Hugging Face or contact our support team.
            """
        )
                

    # --- Submission Tab ---
    with gr.TabItem("🚀 Submit CSV"):
        gr.Markdown("**Submit your prediction CSV file and model metadata.**")
        submission_file = gr.File(label="Upload Prediction CSV", file_types=[".csv"])
        submission_name_textbox = gr.Textbox(label="Name *", placeholder="Enter your name", interactive=True)
        model_description_textbox = gr.Textbox(label="Model *", placeholder="Enter model name", interactive=True)
        link_paper_textbox = gr.Textbox(label="Link to Paper (URL)", placeholder="https://...", interactive=True)
        link_code_textbox = gr.Textbox(label="Link to Source Code (URL)", placeholder="https://...", interactive=True)
        model_type_dropdown = gr.Dropdown(label="Model Type", choices=["Open Source", "Private"], value="Open Source")
        hub_license_dropdown = gr.Dropdown(label="Hub License", choices=["", "MIT", "Apache-2.0", "CC-BY", "CC-BY-NC-4.0", "Other"], value="")

        submit_button = gr.Button("Evaluate Submission")
        evaluation_output = gr.Markdown()
        
        # No explicit username; rate limiting is done via client IP.
        submit_button.click(
            evaluate_and_refresh,
            inputs=[
                submission_file, 
                submission_name_textbox, 
                model_description_textbox,
                link_paper_textbox, 
                link_code_textbox,
                model_type_dropdown,
                hub_license_dropdown
            ],
            outputs=[evaluation_output, leaderboard_output]
        )

    gr.Markdown(
        """
        ### 📙 Citation
        If you use the Food Portion Benchmark dataset in your research, please cite our work as follows:
        ```bibtex
        @misc{foodportionbenchmark2025,
        title={Paper Title},
        author={Authors},
        year={2025},
        note={Under Review}
        }
        ```
        """
    )

# ---------- Scheduler (Optional) ----------
scheduler = BackgroundScheduler()
def restart_space():
    pass
scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.start()

demo.queue(default_concurrency_limit=40).launch(show_api=False)