update

app.py

@@ -1,12 +1,159 @@
 import streamlit as st
-import requests
+import numpy as np
+from keras.models import load_model
+import cv2
+from io import BytesIO
+import mediapipe as mp
 
+# Load the model
+import os
+
+model_path = os.path.abspath('sign_asl_cnn_30_epochs.h5')
+if os.path.exists(model_path):
+    # Load the model
+    model = load_model(model_path)
+else:
+    print(f"File not found: {model_path}")
+class_labels = {i: str(i) if i < 10 else chr(65 + i - 10) for i in range(36)}
+
+# Function to preprocess the image
+def preprocess_image(image):
+    image = cv2.resize(image, (200, 200))
+    image = image / 255.0
+    image = image.reshape(1, 200, 200, 3)
+    return image
+
+# Function to predict the sign language letter
+def predict_letter(image):
+    processed_image = preprocess_image(image)
+    predictions = model.predict(processed_image)
+    predicted_class = np.argmax(predictions, axis=1)[0]
+    sign_letter = class_labels[predicted_class]
+    return sign_letter
+
+# Function to detect hands in the image
+def detect_hands(image):
+    mp_hands = mp.solutions.hands
+    hands = mp_hands.Hands()
+    margin = 15
+
+    # Convert the image to RGB
+    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+    # Process the image and get the hand landmarks
+    results = hands.process(image_rgb)
+
+    if results.multi_hand_landmarks:
+        for landmarks in results.multi_hand_landmarks:
+            # Get bounding box coordinates of the hand
+            landmarks_xy = [(int(landmark.x * image.shape[1]), int(landmark.y * image.shape[0]))
+                            for landmark in landmarks.landmark]
+
+            # Define the bounding box for the hand
+            x_min = max(0, min(landmarks_xy, key=lambda x: x[0])[0] - margin)
+            y_min = max(0, min(landmarks_xy, key=lambda x: x[1])[1] - margin)
+            x_max = min(image.shape[1], max(landmarks_xy, key=lambda x: x[0])[0] + margin)
+            y_max = min(image.shape[0], max(landmarks_xy, key=lambda x: x[1])[1] + margin)
+
+            # Extract the hand region
+            roi = image[y_min:y_max, x_min:x_max]
+
+            # Check if the ROI is empty
+            if roi.size == 0:
+                continue
+
+            # Resize the ROI to match your model's input shape
+            roi = cv2.resize(roi, (200, 200), interpolation=cv2.INTER_AREA)
+            hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)
+
+            lower_yellow = np.array([93, 72, 51])
+            upper_yellow = np.array([224, 194, 183])
+            mask = cv2.inRange(hsv, lower_yellow, upper_yellow)
+            roi = cv2.bitwise_and(roi, roi, mask=mask)
+            roi = roi.reshape(1, 200, 200, 3)  # Ensure it matches your model's input shape
+
+            # Make predictions using your classifier
+            predictions = model.predict(roi)
+            predicted_class = int(np.argmax(predictions, axis=1)[0])
+            result = class_labels[predicted_class]
+
+            # Draw result on the image
+            cv2.putText(image, str(result), (x_min, y_min - 10),
+                        cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
+
+            # Draw bounding box on the image
+            cv2.rectangle(image, (x_min, y_min), (x_max, y_max), (255, 0, 0), 2)
+
+    return image
+
+# Streamlit app
 st.title('Sign Language Recognition')
 
-uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "png"])
-if uploaded_file is not None:
-    if st.button('Predict'):
-        files = {'file': uploaded_file.getvalue()}
-        response = requests.post('http://74.12.105.219:8090/predict', files=files)
-        result = response.json()
-        st.write('Predicted Letter: ', result['predicted_letter'])
+# Sidebar with radio button for Upload/Webcam
+selected_option = st.sidebar.radio("Select Option", ["Upload", "Webcam"], index=0)
+
+if selected_option == "Upload":
+    uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "png"])
+
+    if uploaded_file is not None:
+        if st.button('Predict'):
+            contents = uploaded_file.read()
+            nparr = np.frombuffer(contents, np.uint8)
+            image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+
+            # Make the prediction
+            predicted_letter = predict_letter(image)
+
+            # Display the predicted letter
+            st.write('Predicted Letter:', predicted_letter)
+
+elif selected_option == "Webcam":
+    # Placeholder for webcam frame
+    webcam_frame = st.empty()
+
+    # Placeholder for predicted letter in webcam mode
+    predicted_letter_webcam = st.empty()
+
+    # Placeholder for webcam capture status
+    webcam_capture_status = st.empty()
+
+    # Placeholder for webcam stop button
+    webcam_stop_button = st.empty()
+
+    # Placeholder for webcam status
+    webcam_status = st.empty()
+
+    # Placeholder for webcam button
+    webcam_button = st.button("Start Webcam")
+
+    if webcam_button:
+        webcam_status.text("Webcam is on.")
+        webcam_stop_button = st.button("Stop Webcam")
+
+        # OpenCV video capture
+        cap = cv2.VideoCapture(0)
+
+        while True:
+            # Read the frame from the webcam
+            ret, frame = cap.read()
+
+            # Display the frame in Streamlit
+            webcam_frame.image(frame, channels="BGR")
+
+            # Detect hands in the current frame
+            frame = detect_hands(frame)
+
+            # Convert the frame to JPEG format
+            _, jpeg = cv2.imencode(".jpg", frame)
+
+            # Display the predicted letter
+            predicted_letter = predict_letter(frame)
+            predicted_letter_webcam.text(f"Predicted Letter: {predicted_letter}")
+
+            # Check if the "Stop Webcam" button is clicked
+            if webcam_stop_button:
+                webcam_status.text("Webcam is off.")
+                break
+
+        # Release the webcam when done
+        cap.release()