update: Enhance dataset loading to handle missing columns and streamline data integration

app.py

@@ -61,20 +61,19 @@ def save_reactions_to_dataset(user_type, username, query, results_mpnet, results
         data["reaction"].append(result["reaction"])
 
     try:
+        # Try to load existing dataset
         dataset = load_dataset("HumbleBeeAI/al-ghazali-rag-retrieval-evaluation", split="train")
         existing_data = dataset.to_dict()
+        
+        # Handle missing columns in existing data
+        for key in data:
+            if key not in existing_data:
+                # Add default values for existing rows
+                existing_data[key] = ["" if key in ["user_type", "username", "model_type"] else None] * len(existing_data.get(next(iter(existing_data)), []))
+            existing_data[key].extend(data[key])
     except Exception:
-        existing_data = {
-            "user_type": [],
-            "username": [],
-            "query": [],
-            "retrieved_text": [],
-            "model_type": [],
-            "reaction": []
-        }
-
-    for key in data:
-        existing_data[key].extend(data[key])
+        # If dataset doesn't exist, use the new data structure
+        existing_data = data
 
     updated_dataset = Dataset.from_dict(existing_data)
     updated_dataset.push_to_hub("HumbleBeeAI/al-ghazali-rag-retrieval-evaluation")