Upload app.py

app.py

@@ -3,16 +3,16 @@ import json
 from datetime import datetime
 import os
 import logging
+import random
 
+# Logger setup (unchanged)
 def _setup_logger():
     log_format = logging.Formatter("[%(asctime)s %(levelname)s] %(message)s")
     logger = logging.getLogger()
     logger.setLevel(logging.INFO)
-
     console_handler = logging.StreamHandler()
     console_handler.setFormatter(log_format)
     logger.handlers = [console_handler]
-
     return logger
 
 logger = _setup_logger()
@@ -20,81 +20,40 @@ logger = _setup_logger()
 DATA_DIR = "annotations_data2"
 os.makedirs(DATA_DIR, exist_ok=True)
 
+# Load questions from JSON (unchanged)
 with open("test_pairs2.json", "r") as f:
     response_pairs = json.load(f)
 
+# CSS (unchanged)
 custom_css = """
 @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap');
-
-body {
-    font-family: 'Roboto', sans-serif !important;
-    line-height: 1.6;
-}
-
-.panel {
-    border: 1px solid #e5e7eb !important;
-    border-radius: 12px !important;
-    padding: 20px !important;
-}
-
-button {
-    font-weight: 500 !important;
-    transition: all 0.2s ease !important;
-    font-family: 'Roboto', sans-serif !important;
-}
-
-button:hover {
-    transform: translateY(-1px);
-}
-
-.progress {
-    color: #4f46e5;
-    font-weight: 500;
-}
-
-textarea {
-    border-radius: 8px !important;
-    padding: 12px !important;
-    font-family: 'Roboto', sans-serif !important;
-}
-
-.selected-response {
-    border: 2px solid #4f46e5 !important;
-    background-color: #f5f3ff;
-}
-
-.instruction-panel {
-    background: #f8f9fa !important;
-    border: 1px solid #e0e0e0 !important;
-    border-radius: 12px !important;
-    padding: 25px !important;
-    margin-bottom: 25px !important;
-}
-
-.criteria-list {
-    margin-left: 20px !important;
-    list-style-type: none !important;
-}
-
-.criteria-item {
-    padding: 8px 0 !important;
-}
-
-.highlight {
-    color: #4f46e5;
-    font-weight: 500;
-}
+body { font-family: 'Roboto', sans-serif !important; line-height: 1.6; }
+.panel { border: 1px solid #e5e7eb !important; border-radius: 12px !important; padding: 20px !important; }
+button { font-weight: 500 !important; transition: all 0.2s ease !important; font-family: 'Roboto', sans-serif !important; }
+button:hover { transform: translateY(-1px); }
+.progress { color: #4f46e5; font-weight: 500; }
+textarea { border-radius: 8px !important; padding: 12px !important; font-family: 'Roboto', sans-serif !important; }
+.selected-response { border: 2px solid #4f46e5 !important; background-color: #f5f3ff; }
+.instruction-panel { background: #f8f9fa !important; border: 1px solid #e0e0e0 !important; border-radius: 12px !important; padding: 25px !important; margin-bottom: 25px !important; }
+.criteria-list { margin-left: 20px !important; list-style-type: none !important; }
+.criteria-item { padding: 8px 0 !important; }
+.highlight { color: #4f46e5; font-weight: 500; }
 """
 
+# Updated State class to include selected_indices, form_responses, and forms_completed
 class State:
     def __init__(self):
         self.current_idx = 0
         self.prolific_id = ""
-        self.annotations = []
+        self.selected_indices = []  # List of 40 question indices for this user
+        self.annotations = []       # Annotations for the 40 questions
+        self.form_responses = {}    # Responses to post-test forms
+        self.forms_completed = False  # Flag for form completion
         self.start_time = datetime.now()
 
 state = State()
 
+# Updated save_annotations to include new fields
 def save_annotations():
     if not state.prolific_id:
         return
@@ -102,22 +61,30 @@ def save_annotations():
     filepath = os.path.join(DATA_DIR, filename)
     data = {
         "prolific_id": state.prolific_id,
+        "selected_indices": state.selected_indices,
         "duration": (datetime.now() - state.start_time).total_seconds(),
         "current_idx": state.current_idx,
-        "annotations": state.annotations
+        "annotations": state.annotations,
+        "form_responses": state.form_responses,
+        "forms_completed": state.forms_completed
     }
     with open(filepath, "w") as f:
         json.dump(data, f, indent=2)
     logger.info(f"Saved annotations to {filepath}")
     return filepath
 
+# Updated load_latest_data to load new fields
 def load_latest_data(prolific_id):
     filename = f"{prolific_id}_latest.json"
     filepath = os.path.join(DATA_DIR, filename)
     if os.path.exists(filepath):
         try:
             data = json.load(open(filepath))
-            data["current_idx"] = min(max(data["current_idx"], 0), len(response_pairs)-1)
+            state.selected_indices = data.get("selected_indices", [])
+            state.annotations = data.get("annotations", [])
+            state.form_responses = data.get("form_responses", {})
+            state.forms_completed = data.get("forms_completed", False)
+            state.current_idx = min(max(data.get("current_idx", 0), 0), 39)  # Cap at 39 (0-39 for 40 questions)
             return data
         except Exception as e:
             logger.error(f"Error loading {filepath}: {e}")
@@ -148,21 +115,85 @@ In this task, you'll act as a judge comparing two AI chatbot responses. Your goa
 5. Provide optional feedback and confidence rating
 6. Click "Next" to continue or "Previous" to review
 
-**Note:** You must select a response and confidence level before proceeding to the next question.
-
-*We do not expect any risks beyond what you’d experience in daily life from joining this study. You’ll just read questions and answers, pick your favorite, and rate your confidence—nothing stressful or harmful. It’s as safe as reading a webpage or filling out a short survey.*
+**Note:** You need select a response and confidence level before proceeding to the next question.
 
 *Thank you for contributing to our research! Your input is valuable.*  
 """
 
-MINI_INSTRUCTION = """You’ll compare two AI chatbot answers for different questions and pick the better one. Read the question, then look at Response A and Response B. Choose the one that’s better based on: Helpfulness (answers well, gives useful info), Clarity (clear, logical, on topic), and Emotion (understands feelings, fits the situation).
+MINI_INSTRUCTION = """You’ll compare two AI chatbot answers for different questions and pick the better one. Read the question, then look at Response A and Response B. Choose the one that’s better based on: Perceived Usefulness (answers well, gives useful info), and Social Presence (understands feelings, fits the situation).
 
 *Select your choice and rate your confidence. Click "Next" to move on or "Previous" to go back. You must pick a response and confidence level to continue. Thanks for helping with our research!*
 """
 
+# Define post-test form questions (placeholders; replace with actual questions if available)
+forms_questions = {
+    "Neuro-QoL Cognition Function": [
+        {"question": "In the past 7 days, I had to read something several times to understand it.", "options": ["Never", "Rarely", "Sometimes", "Often", "Very Often"]},
+        {"question": "In the past 7 days, I had to work really hard to pay attention or I would make a mistake.", "options": ["Never", "Rarely", "Sometimes", "Often", "Very Often"]},
+        {"question": "In the past 7 days, I had trouble concentrating.", "options": ["Never", "Rarely", "Sometimes", "Often", "Very Often"]},
+        {"question": "In the past 7 days, I had trouble remembering things.", "options": ["Never", "Rarely", "Sometimes", "Often", "Very Often"]}
+    ],
+    "Wong and Law Emotional Intelligence Scale (WLEIS)": [
+        # // SEA
+        {"question": "I have a good sense of why I have certain feelings most of the time.", "options": ["Strongly Disagree", "Disagree", "Neutral", "Agree", "Strongly Agree"]},
+        {"question": "I have good understanding of my own emotions.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I really understand what I feel.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I always know whether I am happy or not.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        # // OEA
+        {"question": "I always know my friends’ emotions from their behavior.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I am a good observer of others’ emotions.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I am sensitive to the feelings and emotions of others.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I have good understanding of the emotions of people around me.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        # // UOE
+        {"question": "I always set goals for myself and then try my best to achieve them.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I always tell myself I am a competent person.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I am a self-motivated person.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I would always encourage myself to try my best.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        # ROE
+        {"question": "I am able to control my temper and handle difficulties rationally.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I can always calm down quickly when I am very angry.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I have good control of my own emotions.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I can always stay calm in stressful situations.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]}
+    ],
+    "Algorithmic Aversion": [
+        # Trust in LLM
+        {"question": "I trust the answers provided by AI chatbots (e.g., ChatGPT, Grok) to be accurate.", "options": ["Strongly Disagree", "Disagree", "Neutral", "Agree", "Strongly Agree"]},
+        {"question": "I feel confident relying on a AI chatbot for important tasks.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I worry that AI chatbots might give me incorrect information.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+
+        # Preference for Human vs. LLM
+        {"question": "I prefer asking a human expert over a AI chatbot for advice.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I would rather use a human-written article than one generated by a AI chatbot.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I find human interaction more valuable than interacting with a AI chatbot.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+
+        # Willingness to Use LLM
+        {"question": "I would avoid using a AI chatbot if I had other options.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I am willing to use a AI chatbot for daily tasks (e.g., writing, research).", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]},
+        {"question": "I would recommend a AI chatbot to others.", "options": ["Strongly Disagree", "Disagree", "Neutral", " Agree", "Strongly Agree"]}
+    ],
+    "Demographics": [
+        {"question": "What is your highest level of education?", "options": [
+            "Less than high school", 
+            "High school diploma", 
+            "Some college", 
+            "Associate's degree", 
+            "Bachelor's degree", 
+            "Master's degree", 
+            "Doctoral degree"
+        ]},
+        {"question": "How familiar are you with AI chatbots?", "options": [
+            "Not familiar at all", 
+            "Slightly familiar", 
+            "Moderately familiar", 
+            "Very familiar", 
+            "Extremely familiar"
+        ]}
+    ]
+}
+
 def create_interface():
     with gr.Blocks(gr.themes.Ocean(), title="AI Response Evaluation", css=custom_css) as demo:
-        # User ID Section
+        # User ID Section (unchanged layout)
         with gr.Column(visible=True, elem_id="id_section") as id_section:
             with gr.Column(elem_classes="instruction-panel"):
                 gr.Markdown(INSTRUCTION)
@@ -171,13 +202,13 @@ def create_interface():
             prolific_id = gr.Textbox(label="Enter your Prolific ID")
             id_submit_btn = gr.Button("Submit", variant="primary")
 
-        # Main Interface
+        # Main Interface (updated for 40 questions)
         with gr.Column(visible=False, elem_id="main_interface") as main_interface:
-            progress_md = gr.Markdown("**Progress:** 0% (0/0)", elem_classes="progress")
+            progress_md = gr.Markdown("**Progress:** 0% (0/40)", elem_classes="progress")
             gr.HTML('<style>.prompt-highlight { background-color: #e6f7ff; padding: 10px; border: 1px solid #91d5ff; border-radius: 5px; }</style>')
             gr.Markdown(MINI_INSTRUCTION)
             gr.Markdown("---")
-            gr.Markdown("### Current Question")
+            gr.Markdown("### Current Question From a User")
             prompt_box = gr.Markdown(elem_classes="prompt-highlight")
             with gr.Row():
                 with gr.Column(variant="panel"):
@@ -190,7 +221,7 @@ def create_interface():
                 choices=[("Response A", "A"), ("Response B", "B")],
                 label="Select the better response",
             )
-            feedback = gr.Textbox(label="Additional Feedback (optional)", lines=3)
+            feedback = gr.Textbox(label="Additional Feedback (optional)", lines=1)
             confidence = gr.Radio(
                 choices=[("1 - Not confident", 1), ("2", 2), ("3", 3), ("4", 4), ("5 - Very confident", 5)],
                 label="Confidence Rating",
@@ -199,48 +230,85 @@ def create_interface():
                 prev_btn = gr.Button("Previous", variant="secondary")
                 next_btn = gr.Button("Next", variant="primary")
 
-        # Completion Section
+        # New Forms Section
+        with gr.Column(visible=False, elem_id="forms_section") as forms_section:
+            gr.Markdown("## Post-Test Questions")
+            gr.Markdown("Please answer the following questions to complete the study.")
+            form_radios = []
+            for form_name, questions in forms_questions.items():
+                for q in questions:
+                    radio = gr.Radio(choices=q["options"], label=q["question"])
+                    form_radios.append(radio)
+            with gr.Row():
+                back_to_questions_btn = gr.Button("Back to Questions", variant="secondary")
+                submit_forms_btn = gr.Button("Submit Forms", variant="primary")
+
+        # Completion Section (unchanged layout)
         with gr.Column(visible=False, elem_id="completion") as completion_section:
             gr.Markdown("# Thank You!")
             gr.Markdown("### Completion code: `CA7IOI65`")
-            completion_md = gr.Markdown("Your annotations have been saved.")
+            completion_md = gr.Markdown("Your annotations and form responses have been saved.")
             gr.HTML("""
                 <p>Click <a href="https://app.prolific.com/researcher/submissions/complete?cc=CA7IOI65" target="_blank">here</a> to complete the task.</p>
             """)
 
+        # Updated handle_id_submit to assign 40 random questions
         def handle_id_submit(prolific_id_val):
             if not prolific_id_val.strip():
                 raise gr.Error("Please enter a valid Prolific ID")
             state.prolific_id = prolific_id_val.strip()
             data = load_latest_data(state.prolific_id)
-            
             if data:
-                state.annotations = data.get("annotations", [])
-                state.current_idx = data.get("current_idx", 0)
-                if state.current_idx >= len(response_pairs):
-                    save_annotations()
+                if state.forms_completed:
                     return {
                         id_section: gr.update(visible=False),
                         main_interface: gr.update(visible=False),
+                        forms_section: gr.update(visible=False),
                         completion_section: gr.update(visible=True)
                     }
+                elif state.current_idx >= 40:
+                    return {
+                        id_section: gr.update(visible=False),
+                        main_interface: gr.update(visible=False),
+                        forms_section: gr.update(visible=True),
+                        completion_section: gr.update(visible=False)
+                    }
+                else:
+                    return {
+                        id_section: gr.update(visible=False),
+                        main_interface: gr.update(visible=True),
+                        forms_section: gr.update(visible=False),
+                        completion_section: gr.update(visible=False),
+                        **update_interface(state.current_idx)
+                    }
             else:
-                state.annotations = []
+                # New user: assign 40 random questions
+                seed = hash(state.prolific_id) % 1000000
+                random.seed(seed)
+                total_questions = len(response_pairs)
+                if total_questions < 40:
+                    raise ValueError("Not enough questions available")
+                state.selected_indices = random.sample(range(total_questions), 40)
+                state.annotations = [None] * 40
+                state.form_responses = {}
+                state.forms_completed = False
                 state.current_idx = 0
+                return {
+                    id_section: gr.update(visible=False),
+                    main_interface: gr.update(visible=True),
+                    forms_section: gr.update(visible=False),
+                    completion_section: gr.update(visible=False),
+                    **update_interface(0)
+                }
 
-            return {
-                id_section: gr.update(visible=False),
-                main_interface: gr.update(visible=True),
-                completion_section: gr.update(visible=False),
-                **update_interface(state.current_idx)
-            }
-
-        def update_interface(idx):
-            if idx >= len(response_pairs):
-                idx = len(response_pairs) - 1
-            current_data = response_pairs[idx] if idx < len(response_pairs) else {}
-            progress = f"**Progress:** {idx/len(response_pairs):.0%} ({idx}/{len(response_pairs)})"
-            annotation = state.annotations[idx] if idx < len(state.annotations) else None
+        # Updated update_interface to use selected_indices
+        def update_interface(current_idx):
+            if current_idx >= 40:
+                current_idx = 39
+            actual_idx = state.selected_indices[current_idx]
+            current_data = response_pairs[actual_idx]
+            progress = f"**Progress:** {current_idx/40:.0%} ({min(current_idx, 40)}/40)"
+            annotation = state.annotations[current_idx] if current_idx < len(state.annotations) else None
             return {
                 prompt_box: current_data.get("prompt", ""),
                 response_a: current_data.get("responseA", ""),
@@ -251,6 +319,7 @@ def create_interface():
                 selection_radio: annotation["selected"] if annotation else None
             }
 
+        # Updated handle_navigation to transition to forms_section after 40 questions
         def handle_navigation(direction, selection, confidence_val, feedback_val):
             error_msg = None
             if direction == "next":
@@ -258,81 +327,116 @@ def create_interface():
                     error_msg = "Please select a response before proceeding."
                 if not confidence_val:
                     error_msg = "Please select a confidence level before proceeding."
-            
             if error_msg:
                 gr.Warning(error_msg)
                 return {
                     main_interface: gr.update(visible=True),
+                    forms_section: gr.update(visible=False),
                     completion_section: gr.update(visible=False),
                     **update_interface(state.current_idx)
                 }
-    
-            # Save current annotation
             if selection and confidence_val:
+                actual_idx = state.selected_indices[state.current_idx]
                 annotation = {
-                    "id": response_pairs[state.current_idx]["id"],  # Save unique ID
-                    "prompt": response_pairs[state.current_idx]["prompt"],
+                    "id": response_pairs[actual_idx]["id"],
+                    "prompt": response_pairs[actual_idx]["prompt"],
                     "selected": selection,
                     "confidence": confidence_val,
                     "feedback": feedback_val,
                     "timestamp": datetime.now().isoformat()
                 }
-                if state.current_idx < len(state.annotations):
-                    state.annotations[state.current_idx] = annotation
-                else:
-                    state.annotations.append(annotation)
-
-            # Navigation logic
-            try:
-                new_idx = state.current_idx + 1 if direction == "next" else max(0, state.current_idx - 1)
-                state.current_idx = new_idx
-                save_annotations()
-
-                if new_idx >= len(response_pairs):
-                    return {
-                        main_interface: gr.update(visible=False),
-                        completion_section: gr.update(visible=True),
-                        **update_interface(new_idx)
-                    }
-                
+                state.annotations[state.current_idx] = annotation
+            if direction == "next":
+                new_idx = min(state.current_idx + 1, 40)
+            else:
+                new_idx = max(0, state.current_idx - 1)
+            state.current_idx = new_idx
+            save_annotations()
+            if new_idx >= 40:
                 return {
-                    main_interface: gr.update(visible=True),
+                    main_interface: gr.update(visible=False),
+                    forms_section: gr.update(visible=True),
                     completion_section: gr.update(visible=False),
-                    **update_interface(new_idx)
+                    **update_interface(39)  # Keep last question state
                 }
-            
-            except Exception as e:
-                logger.error(f"Navigation error: {e}")
+            else:
                 return {
                     main_interface: gr.update(visible=True),
+                    forms_section: gr.update(visible=False),
                     completion_section: gr.update(visible=False),
-                    **update_interface(state.current_idx)
+                    **update_interface(new_idx)
+                }
+
+        # New function to handle returning to questions from forms
+        def handle_back_to_questions():
+            state.current_idx = 39
+            save_annotations()
+            return {
+                main_interface: gr.update(visible=True),
+                forms_section: gr.update(visible=False),
+                completion_section: gr.update(visible=False),
+                **update_interface(39)
+            }
+
+        # New function to handle form submission
+        def handle_forms_submit(*form_inputs):
+            if any(input_val is None for input_val in form_inputs):
+                gr.Warning("Please answer all questions before submitting.")
+                return {
+                    forms_section: gr.update(visible=True),
+                    completion_section: gr.update(visible=False)
                 }
+            state.form_responses = {}
+            idx = 0
+            for form_name, questions in forms_questions.items():
+                for q in questions:
+                    key = f"{form_name}_{q['question']}"  # Convert tuple to string
+                    state.form_responses[key] = form_inputs[idx]
+                    idx += 1
+            state.forms_completed = True
+            save_annotations()
+            return {
+                forms_section: gr.update(visible=False),
+                completion_section: gr.update(visible=True)
+            }
 
         # Event bindings
         id_submit_btn.click(
             handle_id_submit,
             inputs=prolific_id,
-            outputs=[id_section, main_interface, completion_section, prompt_box, 
-                    response_a, response_b, progress_md, feedback, confidence, selection_radio]
+            outputs=[id_section, main_interface, forms_section, completion_section, prompt_box, 
+                     response_a, response_b, progress_md, feedback, confidence, selection_radio]
         )
 
         prev_btn.click(
             handle_navigation,
             inputs=[gr.State("prev"), selection_radio, confidence, feedback],
-            outputs=[main_interface, completion_section, prompt_box, response_a, 
-                    response_b, progress_md, feedback, confidence, selection_radio]
+            outputs=[main_interface, forms_section, completion_section, prompt_box, response_a, 
+                     response_b, progress_md, feedback, confidence, selection_radio]
         )
 
         next_btn.click(
             handle_navigation,
             inputs=[gr.State("next"), selection_radio, confidence, feedback],
-            outputs=[main_interface, completion_section, prompt_box, response_a,
-                    response_b, progress_md, feedback, confidence, selection_radio]
+            outputs=[main_interface, forms_section, completion_section, prompt_box, response_a,
+                     response_b, progress_md, feedback, confidence, selection_radio]
+        )
+
+        back_to_questions_btn.click(
+            handle_back_to_questions,
+            inputs=[],
+            outputs=[main_interface, forms_section, completion_section, prompt_box, response_a, 
+                     response_b, progress_md, feedback, confidence, selection_radio]
+        )
+
+        submit_forms_btn.click(
+            handle_forms_submit,
+            inputs=form_radios,
+            outputs=[forms_section, completion_section]
         )
 
     return demo
 
 if __name__ == "__main__":
     app = create_interface()
-    app.launch()
+    app.launch()