Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import json | |
| from datetime import datetime | |
| import os | |
| import logging | |
| def _setup_logger(): | |
| log_format = logging.Formatter("[%(asctime)s %(levelname)s] %(message)s") | |
| logger = logging.getLogger() | |
| logger.setLevel(logging.INFO) | |
| console_handler = logging.StreamHandler() | |
| console_handler.setFormatter(log_format) | |
| logger.handlers = [console_handler] | |
| return logger | |
| logger = _setup_logger() | |
| DATA_DIR = "annotations_data2" | |
| os.makedirs(DATA_DIR, exist_ok=True) | |
| with open("test_pairs2.json", "r") as f: | |
| response_pairs = json.load(f) | |
| custom_css = """ | |
| @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap'); | |
| body { | |
| font-family: 'Roboto', sans-serif !important; | |
| line-height: 1.6; | |
| } | |
| .panel { | |
| border: 1px solid #e5e7eb !important; | |
| border-radius: 12px !important; | |
| padding: 20px !important; | |
| } | |
| button { | |
| font-weight: 500 !important; | |
| transition: all 0.2s ease !important; | |
| font-family: 'Roboto', sans-serif !important; | |
| } | |
| button:hover { | |
| transform: translateY(-1px); | |
| } | |
| .progress { | |
| color: #4f46e5; | |
| font-weight: 500; | |
| } | |
| textarea { | |
| border-radius: 8px !important; | |
| padding: 12px !important; | |
| font-family: 'Roboto', sans-serif !important; | |
| } | |
| .selected-response { | |
| border: 2px solid #4f46e5 !important; | |
| background-color: #f5f3ff; | |
| } | |
| .instruction-panel { | |
| background: #f8f9fa !important; | |
| border: 1px solid #e0e0e0 !important; | |
| border-radius: 12px !important; | |
| padding: 25px !important; | |
| margin-bottom: 25px !important; | |
| } | |
| .criteria-list { | |
| margin-left: 20px !important; | |
| list-style-type: none !important; | |
| } | |
| .criteria-item { | |
| padding: 8px 0 !important; | |
| } | |
| .highlight { | |
| color: #4f46e5; | |
| font-weight: 500; | |
| } | |
| """ | |
| class State: | |
| def __init__(self): | |
| self.current_idx = 0 | |
| self.prolific_id = "" | |
| self.annotations = [] | |
| self.start_time = datetime.now() | |
| state = State() | |
| def save_annotations(): | |
| if not state.prolific_id: | |
| return | |
| filename = f"{state.prolific_id}_latest.json" | |
| filepath = os.path.join(DATA_DIR, filename) | |
| data = { | |
| "prolific_id": state.prolific_id, | |
| "duration": (datetime.now() - state.start_time).total_seconds(), | |
| "current_idx": state.current_idx, | |
| "annotations": state.annotations | |
| } | |
| with open(filepath, "w") as f: | |
| json.dump(data, f, indent=2) | |
| logger.info(f"Saved annotations to {filepath}") | |
| return filepath | |
| def load_latest_data(prolific_id): | |
| filename = f"{prolific_id}_latest.json" | |
| filepath = os.path.join(DATA_DIR, filename) | |
| if os.path.exists(filepath): | |
| try: | |
| data = json.load(open(filepath)) | |
| data["current_idx"] = min(max(data["current_idx"], 0), len(response_pairs)-1) | |
| return data | |
| except Exception as e: | |
| logger.error(f"Error loading {filepath}: {e}") | |
| return None | |
| INSTRUCTION = """ | |
| ### Welcome! 🎉 | |
| In this task, you'll act as a judge comparing two AI chatbot responses. Your goal is to determine which response is better based on specific criteria. | |
| ### 📋 Task Overview: | |
| - You'll evaluate multiple questions (prompts), each with two responses (Response A and B) | |
| - Select the better response for each question based on the criteria below | |
| - Your progress will be tracked | |
| ### 🏅 Evaluation Criteria: | |
| 1. **Perceived Usefulness** | |
| → Does the answer address the question effectively and provide relevant information? | |
| 2. **Social Presence** | |
| → Does the answer creates "the feeling of being there with a 'real' person"? | |
| ### 🚀 Getting Started: | |
| 1. Input your Prolific ID to begin | |
| 2. Read the question carefully | |
| 3. Compare both responses side-by-side | |
| 4. Select the better response using the radio buttons | |
| 5. Provide optional feedback and confidence rating | |
| 6. Click "Next" to continue or "Previous" to review | |
| **Note:** You must select a response and confidence level before proceeding to the next question. | |
| *Thank you for contributing to our research! Your input is valuable.* | |
| """ | |
| MINI_INSTRUCTION = """You’ll compare two AI chatbot answers for different questions and pick the better one. Read the question, then look at Response A and Response B. Choose the one that’s better based on: Helpfulness (answers well, gives useful info), Clarity (clear, logical, on topic), and Emotion (understands feelings, fits the situation). | |
| *Select your choice and rate your confidence. Click "Next" to move on or "Previous" to go back. You must pick a response and confidence level to continue. Thanks for helping with our research!* | |
| """ | |
| def create_interface(): | |
| with gr.Blocks(gr.themes.Ocean(), title="AI Response Evaluation", css=custom_css) as demo: | |
| # User ID Section | |
| with gr.Column(visible=True, elem_id="id_section") as id_section: | |
| with gr.Column(elem_classes="instruction-panel"): | |
| gr.Markdown(INSTRUCTION) | |
| gr.Markdown("---") | |
| gr.Markdown("## Prolific ID Verification") | |
| prolific_id = gr.Textbox(label="Enter your Prolific ID") | |
| id_submit_btn = gr.Button("Submit", variant="primary") | |
| # Main Interface | |
| with gr.Column(visible=False, elem_id="main_interface") as main_interface: | |
| progress_md = gr.Markdown("**Progress:** 0% (0/0)", elem_classes="progress") | |
| gr.HTML('<style>.prompt-highlight { background-color: #e6f7ff; padding: 10px; border: 1px solid #91d5ff; border-radius: 5px; }</style>') | |
| gr.Markdown(MINI_INSTRUCTION) | |
| gr.Markdown("---") | |
| gr.Markdown("### Current Question") | |
| prompt_box = gr.Markdown(elem_classes="prompt-highlight") | |
| with gr.Row(): | |
| with gr.Column(variant="panel"): | |
| gr.Markdown("### Response A") | |
| response_a = gr.Markdown(height='200px') | |
| with gr.Column(variant="panel"): | |
| gr.Markdown("### Response B") | |
| response_b = gr.Markdown(height='200px') | |
| selection_radio = gr.Radio( | |
| choices=[("Response A", "A"), ("Response B", "B")], | |
| label="Select the better response", | |
| ) | |
| feedback = gr.Textbox(label="Additional Feedback (optional)", lines=3) | |
| confidence = gr.Radio( | |
| choices=[("1 - Not confident", 1), ("2", 2), ("3", 3), ("4", 4), ("5 - Very confident", 5)], | |
| label="Confidence Rating", | |
| ) | |
| with gr.Row(): | |
| prev_btn = gr.Button("Previous", variant="secondary") | |
| next_btn = gr.Button("Next", variant="primary") | |
| # Completion Section | |
| with gr.Column(visible=False, elem_id="completion") as completion_section: | |
| gr.Markdown("# Thank You!") | |
| gr.Markdown("### Completion code: `CA7IOI65`") | |
| completion_md = gr.Markdown("Your annotations have been saved.") | |
| gr.HTML(""" | |
| <p>Click <a href="https://app.prolific.com/researcher/submissions/complete?cc=CA7IOI65" target="_blank">here</a> to complete the task.</p> | |
| """) | |
| def handle_id_submit(prolific_id_val): | |
| if not prolific_id_val.strip(): | |
| raise gr.Error("Please enter a valid Prolific ID") | |
| state.prolific_id = prolific_id_val.strip() | |
| data = load_latest_data(state.prolific_id) | |
| if data: | |
| state.annotations = data.get("annotations", []) | |
| state.current_idx = data.get("current_idx", 0) | |
| if state.current_idx >= len(response_pairs): | |
| save_annotations() | |
| return { | |
| id_section: gr.update(visible=False), | |
| main_interface: gr.update(visible=False), | |
| completion_section: gr.update(visible=True) | |
| } | |
| else: | |
| state.annotations = [] | |
| state.current_idx = 0 | |
| return { | |
| id_section: gr.update(visible=False), | |
| main_interface: gr.update(visible=True), | |
| completion_section: gr.update(visible=False), | |
| **update_interface(state.current_idx) | |
| } | |
| def update_interface(idx): | |
| if idx >= len(response_pairs): | |
| idx = len(response_pairs) - 1 | |
| current_data = response_pairs[idx] if idx < len(response_pairs) else {} | |
| progress = f"**Progress:** {idx/len(response_pairs):.0%} ({idx}/{len(response_pairs)})" | |
| annotation = state.annotations[idx] if idx < len(state.annotations) else None | |
| return { | |
| prompt_box: current_data.get("prompt", ""), | |
| response_a: current_data.get("responseA", ""), | |
| response_b: current_data.get("responseB", ""), | |
| progress_md: progress, | |
| feedback: annotation["feedback"] if annotation else "", | |
| confidence: annotation["confidence"] if annotation else None, | |
| selection_radio: annotation["selected"] if annotation else None | |
| } | |
| def handle_navigation(direction, selection, confidence_val, feedback_val): | |
| error_msg = None | |
| if direction == "next": | |
| if not selection: | |
| error_msg = "Please select a response before proceeding." | |
| if not confidence_val: | |
| error_msg = "Please select a confidence level before proceeding." | |
| if error_msg: | |
| gr.Warning(error_msg) | |
| return { | |
| main_interface: gr.update(visible=True), | |
| completion_section: gr.update(visible=False), | |
| **update_interface(state.current_idx) | |
| } | |
| # Save current annotation | |
| if selection and confidence_val: | |
| annotation = { | |
| "id": response_pairs[state.current_idx]["id"], # Save unique ID | |
| "prompt": response_pairs[state.current_idx]["prompt"], | |
| "selected": selection, | |
| "confidence": confidence_val, | |
| "feedback": feedback_val, | |
| "timestamp": datetime.now().isoformat() | |
| } | |
| if state.current_idx < len(state.annotations): | |
| state.annotations[state.current_idx] = annotation | |
| else: | |
| state.annotations.append(annotation) | |
| # Navigation logic | |
| try: | |
| new_idx = state.current_idx + 1 if direction == "next" else max(0, state.current_idx - 1) | |
| state.current_idx = new_idx | |
| save_annotations() | |
| if new_idx >= len(response_pairs): | |
| return { | |
| main_interface: gr.update(visible=False), | |
| completion_section: gr.update(visible=True), | |
| **update_interface(new_idx) | |
| } | |
| return { | |
| main_interface: gr.update(visible=True), | |
| completion_section: gr.update(visible=False), | |
| **update_interface(new_idx) | |
| } | |
| except Exception as e: | |
| logger.error(f"Navigation error: {e}") | |
| return { | |
| main_interface: gr.update(visible=True), | |
| completion_section: gr.update(visible=False), | |
| **update_interface(state.current_idx) | |
| } | |
| # Event bindings | |
| id_submit_btn.click( | |
| handle_id_submit, | |
| inputs=prolific_id, | |
| outputs=[id_section, main_interface, completion_section, prompt_box, | |
| response_a, response_b, progress_md, feedback, confidence, selection_radio] | |
| ) | |
| prev_btn.click( | |
| handle_navigation, | |
| inputs=[gr.State("prev"), selection_radio, confidence, feedback], | |
| outputs=[main_interface, completion_section, prompt_box, response_a, | |
| response_b, progress_md, feedback, confidence, selection_radio] | |
| ) | |
| next_btn.click( | |
| handle_navigation, | |
| inputs=[gr.State("next"), selection_radio, confidence, feedback], | |
| outputs=[main_interface, completion_section, prompt_box, response_a, | |
| response_b, progress_md, feedback, confidence, selection_radio] | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| app = create_interface() | |
| app.launch(server_name="0.0.0.0", server_port=7861, share=True) | |