Upload app.py

app.py

@@ -24,7 +24,27 @@ os.makedirs(DATA_DIR, exist_ok=True)
 with open("test_pairs2.json", "r") as f:
     response_pairs = json.load(f)
 
-# CSS (unchanged)
+# Function to generate assignments ensuring each question gets 7 labels
+def generate_assignments(num_questions=120, num_annotators=30, labels_per_question=7, questions_per_annotator=28):
+    assignments = {f"annotator_{i+1}": [] for i in range(num_annotators)}
+    question_assignments = {i: [] for i in range(num_questions)}
+    annotator_capacities = [questions_per_annotator] * num_annotators
+
+    for q in range(num_questions):
+        available_annotators = [(a, annotator_capacities[a]) for a in range(num_annotators) if annotator_capacities[a] > 0]
+        if len(available_annotators) < labels_per_question:
+            raise ValueError(f"Not enough annotators with capacity for question {q}")
+        
+        available_annotators.sort(key=lambda x: x[1], reverse=True)
+        selected_annotators = [a for a, _ in available_annotators[:labels_per_question]]
+        
+        for a in selected_annotators:
+            assignments[f"annotator_{a+1}"].append(q)
+            question_assignments[q].append(a)
+            annotator_capacities[a] -= 1
+
+    return assignments, question_assignments
+
 custom_css = """
 @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap');
 body { font-family: 'Roboto', sans-serif !important; line-height: 1.6; }
@@ -45,13 +65,31 @@ class State:
     def __init__(self):
         self.current_idx = 0
         self.prolific_id = ""
-        self.selected_indices = []  # List of 40 question indices for this user
-        self.annotations = []       # Annotations for the 40 questions
+        self.selected_indices = []  # List of 28 question indices for this user
+        self.annotations = []       # Annotations for the 28 questions
         self.form_responses = {}    # Responses to post-test forms
         self.forms_completed = False  # Flag for form completion
         self.start_time = datetime.now()
 
 state = State()
+ASSIGNED_FILE = "assigned.json"
+
+def load_assigned():
+    if os.path.exists(ASSIGNED_FILE):
+        with open(ASSIGNED_FILE, "r") as f:
+            return json.load(f)
+    return {}
+
+def save_assigned(assigned):
+    with open(ASSIGNED_FILE, "w") as f:
+        json.dump(assigned, f, indent=2)
+
+def get_next_available_assignment(assigned, total_assignments=30):
+    for i in range(1, total_assignments + 1):
+        key = f"annotator_{i}"
+        if key not in assigned.values():
+            return key
+    return None
 
 # Updated save_annotations to include new fields
 def save_annotations():
@@ -61,6 +99,7 @@ def save_annotations():
     filepath = os.path.join(DATA_DIR, filename)
     data = {
         "prolific_id": state.prolific_id,
+        "assignment_key": state.assignment_key,
         "selected_indices": state.selected_indices,
         "duration": (datetime.now() - state.start_time).total_seconds(),
         "current_idx": state.current_idx,
@@ -84,7 +123,7 @@ def load_latest_data(prolific_id):
             state.annotations = data.get("annotations", [])
             state.form_responses = data.get("form_responses", {})
             state.forms_completed = data.get("forms_completed", False)
-            state.current_idx = min(max(data.get("current_idx", 0), 0), 39)  # Cap at 39 (0-39 for 40 questions)
+            state.current_idx = min(max(data.get("current_idx", 0), 0), 27)
             return data
         except Exception as e:
             logger.error(f"Error loading {filepath}: {e}")
@@ -99,6 +138,7 @@ In this task, you'll act as a judge comparing two AI chatbot responses. Your goa
 - You'll evaluate multiple questions (prompts), each with two responses (Response A and B)
 - Select the better response for each question based on the criteria below
 - Your progress will be tracked
+- After completing all questions, you'll answer a few post-test forms
 
 ### 🏅 Evaluation Criteria:
 1. **Perceived Usefulness**  
@@ -194,10 +234,11 @@ def create_interface():
             gr.Markdown("## Prolific ID Verification")
             prolific_id = gr.Textbox(label="Enter your Prolific ID")
             id_submit_btn = gr.Button("Submit", variant="primary")
+            id_message = gr.Markdown("", visible=False)
 
-        # Main Interface (updated for 40 questions)
+        # Main Interface (updated for 28 questions)
         with gr.Column(visible=False, elem_id="main_interface") as main_interface:
-            progress_md = gr.Markdown("**Progress:** 0% (0/40)", elem_classes="progress")
+            progress_md = gr.Markdown("**Progress:** 0% (0/28)", elem_classes="progress")
             gr.HTML('<style>.prompt-highlight { background-color: #e6f7ff; padding: 10px; border: 1px solid #91d5ff; border-radius: 5px; }</style>')
             gr.Markdown(MINI_INSTRUCTION)
             gr.Markdown("---")
@@ -245,26 +286,48 @@ def create_interface():
                 <p>Click <a href="https://app.prolific.com/researcher/submissions/complete?cc=CA7IOI65" target="_blank">here</a> to complete the task.</p>
             """)
 
-        # Updated handle_id_submit to assign 40 random questions
+        # Updated handle_id_submit to assign 28 random questions
         def handle_id_submit(prolific_id_val):
             if not prolific_id_val.strip():
                 raise gr.Error("Please enter a valid Prolific ID")
-            state.prolific_id = prolific_id_val.strip()
-            data = load_latest_data(state.prolific_id)
+            prolific_id = prolific_id_val.strip()
+            assigned = load_assigned()
+            if prolific_id in assigned:
+                assignment_key = assigned[prolific_id]
+            else:
+                next_key = get_next_available_assignment(assigned)
+                if next_key is None:
+                    return {
+                        id_section: gr.update(visible=True),
+                        main_interface: gr.update(visible=False),
+                        forms_section: gr.update(visible=False),
+                        completion_section: gr.update(visible=False),
+                        id_message: gr.update(value="The study is full. Thank you for your interest.", visible=True)
+                    }
+                assigned[prolific_id] = next_key
+                save_assigned(assigned)
+                assignment_key = next_key
+
+            state.prolific_id = prolific_id
+            state.assignment_key = assignment_key
+            state.selected_indices = assignments[assignment_key]
+            data = load_latest_data(prolific_id)
             if data:
                 if state.forms_completed:
                     return {
                         id_section: gr.update(visible=False),
                         main_interface: gr.update(visible=False),
                         forms_section: gr.update(visible=False),
-                        completion_section: gr.update(visible=True)
+                        completion_section: gr.update(visible=True),
+                        id_message: gr.update(value="", visible=False)
                     }
-                elif state.current_idx >= 40:
+                elif state.current_idx >= 28:
                     return {
                         id_section: gr.update(visible=False),
                         main_interface: gr.update(visible=False),
                         forms_section: gr.update(visible=True),
-                        completion_section: gr.update(visible=False)
+                        completion_section: gr.update(visible=False),
+                        id_message: gr.update(value="", visible=False)
                     }
                 else:
                     return {
@@ -272,35 +335,28 @@ def create_interface():
                         main_interface: gr.update(visible=True),
                         forms_section: gr.update(visible=False),
                         completion_section: gr.update(visible=False),
+                        id_message: gr.update(value="", visible=False),
                         **update_interface(state.current_idx)
                     }
             else:
-                # New user: assign 40 random questions
-                seed = hash(state.prolific_id) % 1000000
-                random.seed(seed)
-                total_questions = len(response_pairs)
-                if total_questions < 40:
-                    raise ValueError("Not enough questions available")
-                state.selected_indices = random.sample(range(total_questions), 40)
-                state.annotations = [None] * 40
-                state.form_responses = {}
-                state.forms_completed = False
+                state.annotations = [None] * 28
                 state.current_idx = 0
                 return {
                     id_section: gr.update(visible=False),
                     main_interface: gr.update(visible=True),
                     forms_section: gr.update(visible=False),
                     completion_section: gr.update(visible=False),
+                    id_message: gr.update(value="", visible=False),
                     **update_interface(0)
                 }
 
         # Updated update_interface to use selected_indices
         def update_interface(current_idx):
-            if current_idx >= 40:
+            if current_idx >= 28:
                 current_idx = 39
             actual_idx = state.selected_indices[current_idx]
             current_data = response_pairs[actual_idx]
-            progress = f"**Progress:** {current_idx/40:.0%} ({min(current_idx, 40)}/40)"
+            progress = f"**Progress:** {current_idx/28:.0%} ({min(current_idx, 28)}/28)"
             annotation = state.annotations[current_idx] if current_idx < len(state.annotations) else None
             return {
                 prompt_box: current_data.get("prompt", ""),
@@ -312,7 +368,7 @@ def create_interface():
                 selection_radio: annotation["selected"] if annotation else None
             }
 
-        # Updated handle_navigation to transition to forms_section after 40 questions
+        # Updated handle_navigation to transition to forms_section after 28 questions
         def handle_navigation(direction, selection, confidence_val, feedback_val):
             error_msg = None
             if direction == "next":
@@ -340,12 +396,12 @@ def create_interface():
                 }
                 state.annotations[state.current_idx] = annotation
             if direction == "next":
-                new_idx = min(state.current_idx + 1, 40)
+                new_idx = min(state.current_idx + 1, 28)
             else:
                 new_idx = max(0, state.current_idx - 1)
             state.current_idx = new_idx
             save_annotations()
-            if new_idx >= 40:
+            if new_idx >= 28:
                 return {
                     main_interface: gr.update(visible=False),
                     forms_section: gr.update(visible=True),
@@ -362,13 +418,13 @@ def create_interface():
 
         # New function to handle returning to questions from forms
         def handle_back_to_questions():
-            state.current_idx = 39
+            state.current_idx = 27
             save_annotations()
             return {
                 main_interface: gr.update(visible=True),
                 forms_section: gr.update(visible=False),
                 completion_section: gr.update(visible=False),
-                **update_interface(39)
+                **update_interface(27)
             }
 
         # New function to handle form submission
@@ -397,7 +453,7 @@ def create_interface():
         id_submit_btn.click(
             handle_id_submit,
             inputs=prolific_id,
-            outputs=[id_section, main_interface, forms_section, completion_section, prompt_box, 
+            outputs=[id_section, main_interface, forms_section, completion_section, id_message, prompt_box, 
                      response_a, response_b, progress_md, feedback, confidence, selection_radio]
         )
 
@@ -431,5 +487,14 @@ def create_interface():
     return demo
 
 if __name__ == "__main__":
+    if not os.path.exists("assignments.json"):
+        assignments,_ = generate_assignments()
+        print("Assignments generated.")
+        with open("assignments.json", "w") as f:
+            json.dump(assignments, f, indent=2)
+    else:
+        with open("assignments.json", "r") as f:
+            assignments = json.load(f)
+        print("Assignments loaded.")
     app = create_interface()
     app.launch()