app.py

import gradio as gr
import json
from datetime import datetime
import os
import logging

def _setup_logger():
    log_format = logging.Formatter("[%(asctime)s %(levelname)s] %(message)s")
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    console_handler = logging.StreamHandler()
    console_handler.setFormatter(log_format)
    logger.handlers = [console_handler]

    return logger

logger = _setup_logger()

DATA_DIR = "annotations_data2"
os.makedirs(DATA_DIR, exist_ok=True)

with open("test_pairs2.json", "r") as f:
    response_pairs = json.load(f)

custom_css = """
@import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap');

body {
    font-family: 'Roboto', sans-serif !important;
    line-height: 1.6;
}

.panel {
    border: 1px solid #e5e7eb !important;
    border-radius: 12px !important;
    padding: 20px !important;
}

button {
    font-weight: 500 !important;
    transition: all 0.2s ease !important;
    font-family: 'Roboto', sans-serif !important;
}

button:hover {
    transform: translateY(-1px);
}

.progress {
    color: #4f46e5;
    font-weight: 500;
}

textarea {
    border-radius: 8px !important;
    padding: 12px !important;
    font-family: 'Roboto', sans-serif !important;
}

.selected-response {
    border: 2px solid #4f46e5 !important;
    background-color: #f5f3ff;
}

.instruction-panel {
    background: #f8f9fa !important;
    border: 1px solid #e0e0e0 !important;
    border-radius: 12px !important;
    padding: 25px !important;
    margin-bottom: 25px !important;
}

.criteria-list {
    margin-left: 20px !important;
    list-style-type: none !important;
}

.criteria-item {
    padding: 8px 0 !important;
}

.highlight {
    color: #4f46e5;
    font-weight: 500;
}
"""

class State:
    def __init__(self):
        self.current_idx = 0
        self.prolific_id = ""
        self.annotations = []
        self.start_time = datetime.now()

state = State()

def save_annotations():
    if not state.prolific_id:
        return
    filename = f"{state.prolific_id}_latest.json"
    filepath = os.path.join(DATA_DIR, filename)
    data = {
        "prolific_id": state.prolific_id,
        "duration": (datetime.now() - state.start_time).total_seconds(),
        "current_idx": state.current_idx,
        "annotations": state.annotations
    }
    with open(filepath, "w") as f:
        json.dump(data, f, indent=2)
    logger.info(f"Saved annotations to {filepath}")
    return filepath

def load_latest_data(prolific_id):
    filename = f"{prolific_id}_latest.json"
    filepath = os.path.join(DATA_DIR, filename)
    if os.path.exists(filepath):
        try:
            data = json.load(open(filepath))
            data["current_idx"] = min(max(data["current_idx"], 0), len(response_pairs)-1)
            return data
        except Exception as e:
            logger.error(f"Error loading {filepath}: {e}")
    return None

INSTRUCTION = """
### Welcome! 🎉

In this task, you'll act as a judge comparing two AI chatbot responses. Your goal is to determine which response is better based on specific criteria.

### 📋 Task Overview:
- You'll evaluate multiple questions (prompts), each with two responses (Response A and B)
- Select the better response for each question based on the criteria below
- Your progress will be tracked

### 🏅 Evaluation Criteria:
1. **Perceived Usefulness**  
   → Does the answer address the question effectively and provide relevant information?
2. **Social Presence**  
   → Does the answer creates "the feeling of being there with a 'real' person"?


### 🚀 Getting Started:
1. Input your Prolific ID to begin
2. Read the question carefully
3. Compare both responses side-by-side
4. Select the better response using the radio buttons
5. Provide optional feedback and confidence rating
6. Click "Next" to continue or "Previous" to review

**Note:** You must select a response and confidence level before proceeding to the next question.

*Thank you for contributing to our research! Your input is valuable.*  
"""

MINI_INSTRUCTION = """You’ll compare two AI chatbot answers for different questions and pick the better one. Read the question, then look at Response A and Response B. Choose the one that’s better based on: Helpfulness (answers well, gives useful info), Clarity (clear, logical, on topic), and Emotion (understands feelings, fits the situation).

*Select your choice and rate your confidence. Click "Next" to move on or "Previous" to go back. You must pick a response and confidence level to continue. Thanks for helping with our research!*
"""

def create_interface():
    with gr.Blocks(gr.themes.Ocean(), title="AI Response Evaluation", css=custom_css) as demo:
        # User ID Section
        with gr.Column(visible=True, elem_id="id_section") as id_section:
            with gr.Column(elem_classes="instruction-panel"):
                gr.Markdown(INSTRUCTION)
            gr.Markdown("---")
            gr.Markdown("## Prolific ID Verification")
            prolific_id = gr.Textbox(label="Enter your Prolific ID")
            id_submit_btn = gr.Button("Submit", variant="primary")

        # Main Interface
        with gr.Column(visible=False, elem_id="main_interface") as main_interface:
            progress_md = gr.Markdown("**Progress:** 0% (0/0)", elem_classes="progress")
            gr.HTML('<style>.prompt-highlight { background-color: #e6f7ff; padding: 10px; border: 1px solid #91d5ff; border-radius: 5px; }</style>')
            gr.Markdown(MINI_INSTRUCTION)
            gr.Markdown("---")
            gr.Markdown("### Current Question")
            prompt_box = gr.Markdown(elem_classes="prompt-highlight")
            with gr.Row():
                with gr.Column(variant="panel"):
                    gr.Markdown("### Response A")
                    response_a = gr.Markdown(height='200px')
                with gr.Column(variant="panel"):
                    gr.Markdown("### Response B")
                    response_b = gr.Markdown(height='200px')
            selection_radio = gr.Radio(
                choices=[("Response A", "A"), ("Response B", "B")],
                label="Select the better response",
            )
            feedback = gr.Textbox(label="Additional Feedback (optional)", lines=3)
            confidence = gr.Radio(
                choices=[("1 - Not confident", 1), ("2", 2), ("3", 3), ("4", 4), ("5 - Very confident", 5)],
                label="Confidence Rating",
            )
            with gr.Row():
                prev_btn = gr.Button("Previous", variant="secondary")
                next_btn = gr.Button("Next", variant="primary")

        # Completion Section
        with gr.Column(visible=False, elem_id="completion") as completion_section:
            gr.Markdown("# Thank You!")
            gr.Markdown("### Completion code: `CA7IOI65`")
            completion_md = gr.Markdown("Your annotations have been saved.")
            gr.HTML("""
                <p>Click <a href="https://app.prolific.com/researcher/submissions/complete?cc=CA7IOI65" target="_blank">here</a> to complete the task.</p>
            """)

        def handle_id_submit(prolific_id_val):
            if not prolific_id_val.strip():
                raise gr.Error("Please enter a valid Prolific ID")
            state.prolific_id = prolific_id_val.strip()
            data = load_latest_data(state.prolific_id)
            
            if data:
                state.annotations = data.get("annotations", [])
                state.current_idx = data.get("current_idx", 0)
                if state.current_idx >= len(response_pairs):
                    save_annotations()
                    return {
                        id_section: gr.update(visible=False),
                        main_interface: gr.update(visible=False),
                        completion_section: gr.update(visible=True)
                    }
            else:
                state.annotations = []
                state.current_idx = 0

            return {
                id_section: gr.update(visible=False),
                main_interface: gr.update(visible=True),
                completion_section: gr.update(visible=False),
                **update_interface(state.current_idx)
            }

        def update_interface(idx):
            if idx >= len(response_pairs):
                idx = len(response_pairs) - 1
            current_data = response_pairs[idx] if idx < len(response_pairs) else {}
            progress = f"**Progress:** {idx/len(response_pairs):.0%} ({idx}/{len(response_pairs)})"
            annotation = state.annotations[idx] if idx < len(state.annotations) else None
            return {
                prompt_box: current_data.get("prompt", ""),
                response_a: current_data.get("responseA", ""),
                response_b: current_data.get("responseB", ""),
                progress_md: progress,
                feedback: annotation["feedback"] if annotation else "",
                confidence: annotation["confidence"] if annotation else None,
                selection_radio: annotation["selected"] if annotation else None
            }

        def handle_navigation(direction, selection, confidence_val, feedback_val):
            error_msg = None
            if direction == "next":
                if not selection:
                    error_msg = "Please select a response before proceeding."
                if not confidence_val:
                    error_msg = "Please select a confidence level before proceeding."
            
            if error_msg:
                gr.Warning(error_msg)
                return {
                    main_interface: gr.update(visible=True),
                    completion_section: gr.update(visible=False),
                    **update_interface(state.current_idx)
                }
    
            # Save current annotation
            if selection and confidence_val:
                annotation = {
                    "id": response_pairs[state.current_idx]["id"],  # Save unique ID
                    "prompt": response_pairs[state.current_idx]["prompt"],
                    "selected": selection,
                    "confidence": confidence_val,
                    "feedback": feedback_val,
                    "timestamp": datetime.now().isoformat()
                }
                if state.current_idx < len(state.annotations):
                    state.annotations[state.current_idx] = annotation
                else:
                    state.annotations.append(annotation)

            # Navigation logic
            try:
                new_idx = state.current_idx + 1 if direction == "next" else max(0, state.current_idx - 1)
                state.current_idx = new_idx
                save_annotations()

                if new_idx >= len(response_pairs):
                    return {
                        main_interface: gr.update(visible=False),
                        completion_section: gr.update(visible=True),
                        **update_interface(new_idx)
                    }
                
                return {
                    main_interface: gr.update(visible=True),
                    completion_section: gr.update(visible=False),
                    **update_interface(new_idx)
                }
            
            except Exception as e:
                logger.error(f"Navigation error: {e}")
                return {
                    main_interface: gr.update(visible=True),
                    completion_section: gr.update(visible=False),
                    **update_interface(state.current_idx)
                }

        # Event bindings
        id_submit_btn.click(
            handle_id_submit,
            inputs=prolific_id,
            outputs=[id_section, main_interface, completion_section, prompt_box, 
                    response_a, response_b, progress_md, feedback, confidence, selection_radio]
        )

        prev_btn.click(
            handle_navigation,
            inputs=[gr.State("prev"), selection_radio, confidence, feedback],
            outputs=[main_interface, completion_section, prompt_box, response_a, 
                    response_b, progress_md, feedback, confidence, selection_radio]
        )

        next_btn.click(
            handle_navigation,
            inputs=[gr.State("next"), selection_radio, confidence, feedback],
            outputs=[main_interface, completion_section, prompt_box, response_a,
                    response_b, progress_md, feedback, confidence, selection_radio]
        )

    return demo

if __name__ == "__main__":
    app = create_interface()
    app.launch(server_name="0.0.0.0", server_port=7861, share=True)