updating app

app.py

@@ -1,65 +1,477 @@
 import streamlit as st
-from transformers import pipeline
-from PIL import Image
 import cv2
 import numpy as np
+import time
+import plotly.graph_objects as go
+from transformers import pipeline
+from PIL import Image
 import torch
+from collections import deque
+
+# Set page config
+st.set_page_config(
+    page_title="Real-Time Emotion Detection",
+    page_icon="😀",
+    layout="wide"
+)
 
 # --- App Title and Description ---
-st.title("Real-Time Emotion Detection App")
+st.title("Advanced Real-Time Emotion Detection")
 st.write("""
-This app uses a lightweight, pre-trained emotion detection model from Hugging Face to predict emotions 
-from faces in an image. You can either upload an image or use your webcam to capture an image.
+This app detects emotions in real-time using your webcam. It tracks facial expressions continuously
+and provides visual feedback on detected emotions.
 """)
 
-# --- Load the Emotion Detection Model ---
-# Cache the model loading so it isn’t reloaded on every app interaction.
+# --- Load Models ---
 @st.cache_resource(show_spinner=False)
-def load_emotion_detector():
-    # Loads the Hugging Face image-classification pipeline with the specified model.
-    classifier = pipeline("image-classification", model="dima806/facial_emotions_image_detection")
+def load_emotion_detector(model_name="dima806/facial_emotions_image_detection"):
+    """Load the emotion detection model."""
+    with st.spinner(f"Loading emotion detection model ({model_name})..."):
+        classifier = pipeline("image-classification", model=model_name)
     return classifier
 
-classifier = load_emotion_detector()
+@st.cache_resource(show_spinner=False)
+def load_face_detector():
+    """Load the face detector model."""
+    with st.spinner("Loading face detection model..."):
+        # Load OpenCV's face detector
+        face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
+    return face_cascade
+
+# --- Sidebar: Model and Settings ---
+st.sidebar.header("Settings")
+
+# Model selection
+model_options = {
+    "Facial Emotions (Default)": "dima806/facial_emotions_image_detection",
+    "Facial Expressions": "juliensimon/distilbert-emotion"
+}
+selected_model = st.sidebar.selectbox(
+    "Choose Emotion Model",
+    list(model_options.keys())
+)
+
+# Input method selection
+input_method = st.sidebar.radio(
+    "Choose Input Method",
+    ["Real-time Webcam", "Upload an Image", "Capture Image"]
+)
+
+# Confidence threshold
+confidence_threshold = st.sidebar.slider(
+    "Confidence Threshold",
+    min_value=0.0,
+    max_value=1.0,
+    value=0.5,
+    step=0.05
+)
+
+# Face detection toggle
+use_face_detection = st.sidebar.checkbox("Enable Face Detection", value=True)
+
+# History length for real-time tracking
+if input_method == "Real-time Webcam":
+    history_length = st.sidebar.slider(
+        "Emotion History Length (seconds)",
+        min_value=5,
+        max_value=60,
+        value=10,
+        step=5
+    )
 
-# --- Sidebar: Select Input Method ---
-st.sidebar.header("Select Input Method")
-input_method = st.sidebar.radio("Choose one:", ["Upload an Image", "Capture with Webcam"])
+# Load the selected model
+classifier = load_emotion_detector(model_options[selected_model])
+face_detector = load_face_detector()
 
-# --- Process Image and Perform Inference ---
-def predict_emotion(image: Image.Image):
-    # Optionally, you can perform additional preprocessing (e.g., face detection or cropping) here.
-    results = classifier(image)
-    # The pipeline returns a list of dictionaries sorted by score.
-    top_prediction = results[0]
-    return top_prediction
+# --- Utility Functions ---
+def detect_faces(image):
+    """Detect faces in an image using OpenCV."""
+    # Convert PIL Image to OpenCV format
+    if isinstance(image, Image.Image):
+        opencv_image = np.array(image)
+        opencv_image = opencv_image[:, :, ::-1].copy()  # Convert RGB to BGR
+    else:
+        opencv_image = image
+    
+    # Convert to grayscale for face detection
+    gray = cv2.cvtColor(opencv_image, cv2.COLOR_BGR2GRAY)
+    
+    # Detect faces
+    faces = face_detector.detectMultiScale(
+        gray,
+        scaleFactor=1.1,
+        minNeighbors=5,
+        minSize=(30, 30)
+    )
+    
+    return faces, opencv_image
 
-# --- Main Section: Handling Input Methods ---
+def process_image_for_emotion(image, face=None):
+    """Process image for emotion detection."""
+    if isinstance(image, np.ndarray):
+        # Convert OpenCV image to PIL
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        image = Image.fromarray(image)
+    
+    if face is not None:
+        # Crop to face region
+        x, y, w, h = face
+        image = image.crop((x, y, x+w, y+h))
+    
+    return image
 
+def predict_emotion(image):
+    """Predict emotion from an image."""
+    try:
+        results = classifier(image)
+        return results[0]  # Return top prediction
+    except Exception as e:
+        st.error(f"Error during emotion prediction: {str(e)}")
+        return {"label": "Error", "score": 0.0}
+
+def draw_faces_with_emotions(image, faces, emotions):
+    """Draw rectangles around faces and label with emotions."""
+    img = image.copy()
+    
+    # Define colors for different emotions (BGR format)
+    emotion_colors = {
+        "happy": (0, 255, 0),      # Green
+        "sad": (255, 0, 0),        # Blue
+        "neutral": (255, 255, 0),  # Cyan
+        "angry": (0, 0, 255),      # Red
+        "surprise": (255, 165, 0), # Orange
+        "fear": (128, 0, 128),     # Purple
+        "disgust": (0, 128, 128)   # Brown
+    }
+    
+    # Default color for unknown emotions
+    default_color = (255, 255, 255)  # White
+    
+    for (x, y, w, h), emotion in zip(faces, emotions):
+        # Get color based on emotion (lowercase and remove any prefix)
+        emotion_key = emotion["label"].lower().split("_")[-1]
+        color = emotion_colors.get(emotion_key, default_color)
+        
+        # Draw rectangle around face
+        cv2.rectangle(img, (x, y), (x+w, y+h), color, 2)
+        
+        # Add emotion label and confidence
+        label = f"{emotion['label']} ({emotion['score']:.2f})"
+        cv2.putText(img, label, (x, y-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
+    
+    return img
+
+# --- Main App Logic ---
 if input_method == "Upload an Image":
     uploaded_file = st.file_uploader("Choose an image file", type=["jpg", "jpeg", "png"])
+    
     if uploaded_file is not None:
-        # Open the image file with PIL.
+        # Load and display image
         image = Image.open(uploaded_file).convert("RGB")
-        st.image(image, caption="Uploaded Image", use_column_width=True)
-        prediction = predict_emotion(image)
-        st.subheader("Prediction:")
-        st.write(f"**Emotion:** {prediction['label']}")
-        st.write(f"**Confidence:** {prediction['score']:.2f}")
         
-elif input_method == "Capture with Webcam":
-    # st.camera_input returns an image file-like object when a picture is taken.
+        col1, col2 = st.columns(2)
+        with col1:
+            st.image(image, caption="Uploaded Image", use_column_width=True)
+        
+        # Process image
+        if use_face_detection:
+            faces, opencv_image = detect_faces(image)
+            
+            if len(faces) > 0:
+                emotions = []
+                for face in faces:
+                    face_img = process_image_for_emotion(image, face)
+                    emotions.append(predict_emotion(face_img))
+                
+                # Draw faces with emotions
+                result_image = draw_faces_with_emotions(opencv_image, faces, emotions)
+                
+                with col2:
+                    st.image(result_image, caption="Detected Emotions", channels="BGR", use_column_width=True)
+                
+                # Display predictions
+                st.subheader("Detected Emotions:")
+                for i, (emotion, face) in enumerate(zip(emotions, faces)):
+                    if emotion["score"] >= confidence_threshold:
+                        st.write(f"Face {i+1}: **{emotion['label']}** (Confidence: {emotion['score']:.2f})")
+                        
+                        # Show confidence bars
+                        top_emotions = classifier(process_image_for_emotion(image, face))
+                        labels = [item["label"] for item in top_emotions]
+                        scores = [item["score"] for item in top_emotions]
+                        
+                        fig = go.Figure(go.Bar(
+                            x=scores,
+                            y=labels,
+                            orientation='h'
+                        ))
+                        fig.update_layout(
+                            title=f"Emotion Confidence - Face {i+1}",
+                            xaxis_title="Confidence",
+                            yaxis_title="Emotion",
+                            height=300
+                        )
+                        st.plotly_chart(fig, use_container_width=True)
+            else:
+                st.warning("No faces detected in the image. Try another image or disable face detection.")
+        else:
+            # Process the whole image
+            prediction = predict_emotion(image)
+            st.subheader("Prediction:")
+            st.write(f"**Emotion:** {prediction['label']}")
+            st.write(f"**Confidence:** {prediction['score']:.2f}")
+
+elif input_method == "Capture Image":
     picture = st.camera_input("Capture an Image")
+    
     if picture is not None:
-        # Load image from the captured file.
         image = Image.open(picture).convert("RGB")
-        st.image(image, caption="Captured Image", use_column_width=True)
-        prediction = predict_emotion(image)
-        st.subheader("Prediction:")
-        st.write(f"**Emotion:** {prediction['label']}")
-        st.write(f"**Confidence:** {prediction['score']:.2f}")
+        
+        col1, col2 = st.columns(2)
+        with col1:
+            st.image(image, caption="Captured Image", use_column_width=True)
+        
+        # Process image
+        if use_face_detection:
+            faces, opencv_image = detect_faces(image)
+            
+            if len(faces) > 0:
+                emotions = []
+                for face in faces:
+                    face_img = process_image_for_emotion(image, face)
+                    emotions.append(predict_emotion(face_img))
+                
+                # Draw faces with emotions
+                result_image = draw_faces_with_emotions(opencv_image, faces, emotions)
+                
+                with col2:
+                    st.image(result_image, caption="Detected Emotions", channels="BGR", use_column_width=True)
+                
+                # Display predictions
+                st.subheader("Detected Emotions:")
+                for i, (emotion, face) in enumerate(zip(emotions, faces)):
+                    if emotion["score"] >= confidence_threshold:
+                        st.write(f"Face {i+1}: **{emotion['label']}** (Confidence: {emotion['score']:.2f})")
+            else:
+                st.warning("No faces detected in the image. Try another image or disable face detection.")
+        else:
+            # Process the whole image
+            prediction = predict_emotion(image)
+            st.subheader("Prediction:")
+            st.write(f"**Emotion:** {prediction['label']}")
+            st.write(f"**Confidence:** {prediction['score']:.2f}")
 
-# --- Optional: Additional Instructions ---
-st.write("""
-*Note: For best results in real-time detection, consider focusing the camera on your face or uploading a clear face image.*
-""")
+elif input_method == "Real-time Webcam":
+    st.subheader("Real-time Emotion Detection")
+    st.write("Click 'Start' to begin real-time emotion detection using your webcam.")
+    
+    # Create a placeholder for the webcam feed
+    video_placeholder = st.empty()
+    
+    # Create a placeholder for metrics
+    metrics_placeholder = st.empty()
+    
+    # Create a placeholder for emotion history chart
+    chart_placeholder = st.empty()
+    
+    # Initialize session state for tracking emotions over time
+    if 'emotion_history' not in st.session_state:
+        st.session_state.emotion_history = {}
+        st.session_state.last_update_time = time.time()
+        st.session_state.frame_count = 0
+    
+    # Start/Stop button
+    start_button = st.button("Start" if 'running' not in st.session_state or not st.session_state.running else "Stop")
+    
+    if start_button:
+        st.session_state.running = not st.session_state.get('running', False)
+    
+    # If running, capture and process webcam feed
+    if st.session_state.get('running', False):
+        try:
+            # Open the webcam
+            cap = cv2.VideoCapture(0)
+            
+            # Check if webcam opened successfully
+            if not cap.isOpened():
+                st.error("Could not open webcam. Please check your camera settings.")
+                st.session_state.running = False
+            else:
+                # Create deques for tracking emotions
+                emotion_deques = {}
+                timestamp_deque = deque(maxlen=30*history_length)  # Store timestamps for X seconds at 30fps
+                
+                while st.session_state.get('running', False):
+                    # Read frame
+                    ret, frame = cap.read()
+                    
+                    if not ret:
+                        st.error("Failed to capture frame from webcam")
+                        break
+                    
+                    # Flip the frame horizontally for a more natural view
+                    frame = cv2.flip(frame, 1)
+                    
+                    # Increment frame count for FPS calculation
+                    st.session_state.frame_count += 1
+                    
+                    # Detect faces
+                    if use_face_detection:
+                        faces, _ = detect_faces(frame)
+                        
+                        if len(faces) > 0:
+                            # Process each face
+                            emotions = []
+                            for face in faces:
+                                face_img = process_image_for_emotion(frame, face)
+                                emotions.append(predict_emotion(face_img))
+                            
+                            # Draw faces with emotions
+                            frame = draw_faces_with_emotions(frame, faces, emotions)
+                            
+                            # Update emotion history
+                            current_time = time.time()
+                            timestamp_deque.append(current_time)
+                            
+                            for i, emotion in enumerate(emotions):
+                                if emotion["score"] >= confidence_threshold:
+                                    face_id = f"Face {i+1}"
+                                    if face_id not in emotion_deques:
+                                        emotion_deques[face_id] = deque(maxlen=30*history_length)
+                                    
+                                    emotion_deques[face_id].append({
+                                        "emotion": emotion["label"],
+                                        "confidence": emotion["score"],
+                                        "time": current_time
+                                    })
+                        else:
+                            # No faces detected
+                            pass
+                    else:
+                        # Process the whole frame
+                        pil_image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+                        emotion = predict_emotion(pil_image)
+                        
+                        # Display emotion on frame
+                        cv2.putText(
+                            frame,
+                            f"{emotion['label']} ({emotion['score']:.2f})",
+                            (10, 30),
+                            cv2.FONT_HERSHEY_SIMPLEX,
+                            1,
+                            (0, 255, 0),
+                            2
+                        )
+                        
+                        # Update emotion history
+                        current_time = time.time()
+                        timestamp_deque.append(current_time)
+                        
+                        if "Frame" not in emotion_deques:
+                            emotion_deques["Frame"] = deque(maxlen=30*history_length)
+                        
+                        emotion_deques["Frame"].append({
+                            "emotion": emotion["label"],
+                            "confidence": emotion["score"],
+                            "time": current_time
+                        })
+                    
+                    # Calculate FPS
+                    current_time = time.time()
+                    time_diff = current_time - st.session_state.last_update_time
+                    if time_diff >= 1.0:  # Update every second
+                        fps = st.session_state.frame_count / time_diff
+                        st.session_state.last_update_time = current_time
+                        st.session_state.frame_count = 0
+                        
+                        # Update metrics
+                        with metrics_placeholder.container():
+                            cols = st.columns(3)
+                            cols[0].metric("FPS", f"{fps:.1f}")
+                            cols[1].metric("Faces Detected", len(faces) if use_face_detection else "N/A")
+                    
+                    # Display the frame
+                    video_placeholder.image(frame, channels="BGR", use_column_width=True)
+                    
+                    # Update emotion history chart periodically
+                    if len(timestamp_deque) > 0 and time_diff >= 0.5:  # Update chart every 0.5 seconds
+                        with chart_placeholder.container():
+                            # Create tabs for each face
+                            if len(emotion_deques) > 0:
+                                tabs = st.tabs(list(emotion_deques.keys()))
+                                
+                                for i, (face_id, emotion_data) in enumerate(emotion_deques.items()):
+                                    with tabs[i]:
+                                        if len(emotion_data) > 0:
+                                            # Count occurrences of each emotion
+                                            emotion_counts = {}
+                                            for entry in emotion_data:
+                                                emotion = entry["emotion"]
+                                                if emotion not in emotion_counts:
+                                                    emotion_counts[emotion] = 0
+                                                emotion_counts[emotion] += 1
+                                            
+                                            # Create pie chart for emotion distribution
+                                            fig = go.Figure(data=[go.Pie(
+                                                labels=list(emotion_counts.keys()),
+                                                values=list(emotion_counts.values()),
+                                                hole=.3
+                                            )])
+                                            fig.update_layout(title=f"Emotion Distribution - {face_id}")
+                                            st.plotly_chart(fig, use_container_width=True)
+                                            
+                                            # Create line chart for emotion confidence over time
+                                            emotions = list(emotion_data)[-20:]  # Get the last 20 entries
+                                            times = [(e["time"] - emotions[0]["time"]) for e in emotions]
+                                            confidences = [e["confidence"] for e in emotions]
+                                            emotion_labels = [e["emotion"] for e in emotions]
+                                            
+                                            fig = go.Figure()
+                                            fig.add_trace(go.Scatter(
+                                                x=times,
+                                                y=confidences,
+                                                mode='lines+markers',
+                                                text=emotion_labels,
+                                                hoverinfo='text+y'
+                                            ))
+                                            fig.update_layout(
+                                                title=f"Emotion Confidence Over Time - {face_id}",
+                                                xaxis_title="Time (seconds)",
+                                                yaxis_title="Confidence",
+                                                yaxis=dict(range=[0, 1])
+                                            )
+                                            st.plotly_chart(fig, use_container_width=True)
+                                        else:
+                                            st.info(f"No emotion data available for {face_id} yet.")
+                            else:
+                                st.info("No emotion data available yet.")
+                
+                # Release the webcam when done
+                cap.release()
+        
+        except Exception as e:
+            st.error(f"Error during webcam processing: {str(e)}")
+            st.session_state.running = False
+    else:
+        # Display a placeholder image when not running
+        placeholder_img = np.zeros((300, 500, 3), dtype=np.uint8)
+        cv2.putText(
+            placeholder_img,
+            "Click 'Start' to begin",
+            (80, 150),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            1,
+            (255, 255, 255),
+            2
+        )
+        video_placeholder.image(placeholder_img, channels="BGR", use_column_width=True)
+
+# --- Footer ---
+st.markdown("---")
+st.markdown("""
+**Tips for Best Results:**
+- Ensure good lighting for accurate face detection
+- Position your face clearly in the frame
+- Try different emotion models for comparison
+- Adjust the confidence threshold if emotions aren't being detected
+""")

requirements.txt

@@ -2,4 +2,5 @@ streamlit
 transformers
 pillow
 opencv-python
-torch
+torch
+plotly