Spaces:
Sleeping
Sleeping
Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,656 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# -*- coding: utf-8 -*-
|
2 |
+
import streamlit as st
|
3 |
+
import os
|
4 |
+
import asyncio
|
5 |
+
import base64
|
6 |
+
import io
|
7 |
+
import threading
|
8 |
+
import traceback
|
9 |
+
import atexit
|
10 |
+
import time
|
11 |
+
import logging
|
12 |
+
from dotenv import load_dotenv
|
13 |
+
|
14 |
+
import cv2
|
15 |
+
import pyaudio
|
16 |
+
import PIL.Image
|
17 |
+
|
18 |
+
# Import websockets for explicit exception handling
|
19 |
+
import websockets.exceptions
|
20 |
+
|
21 |
+
from google import genai
|
22 |
+
from google.genai import types
|
23 |
+
|
24 |
+
from streamlit_webrtc import (
|
25 |
+
webrtc_streamer,
|
26 |
+
WebRtcMode,
|
27 |
+
AudioProcessorBase,
|
28 |
+
VideoProcessorBase,
|
29 |
+
)
|
30 |
+
|
31 |
+
load_dotenv()
|
32 |
+
|
33 |
+
|
34 |
+
# Audio configuration - fix audio format issues
|
35 |
+
FORMAT = pyaudio.paInt16
|
36 |
+
CHANNELS = 1
|
37 |
+
SEND_SAMPLE_RATE = 16000 # Changed to match mime_type for consistency
|
38 |
+
RECEIVE_SAMPLE_RATE = 16000 # Changed from 24000 to 16000 to match send rate
|
39 |
+
CHUNK_SIZE = 1024
|
40 |
+
|
41 |
+
# Map PyAudio format to a more descriptive name for clarity.
|
42 |
+
PYAUDIO_FORMAT = FORMAT # pyaudio.paInt16
|
43 |
+
PYAUDIO_CHANNELS = CHANNELS
|
44 |
+
PYAUDIO_PLAYBACK_CHUNK_SIZE = CHUNK_SIZE
|
45 |
+
GEMINI_AUDIO_RECEIVE_SAMPLE_RATE = RECEIVE_SAMPLE_RATE
|
46 |
+
|
47 |
+
# Video configuration
|
48 |
+
VIDEO_FPS_TO_GEMINI = 1 # Reduced from 2 to lower bandwidth
|
49 |
+
VIDEO_API_RESIZE = (512, 512) # Reduced from 1024x1024 to lower payload size
|
50 |
+
MAX_PAYLOAD_SIZE_BYTES = 60000 # Just under 64KB WebSocket limit
|
51 |
+
|
52 |
+
# Queue sizes
|
53 |
+
MEDIA_TO_GEMINI_QUEUE_MAXSIZE = 10
|
54 |
+
AUDIO_PLAYBACK_QUEUE_MAXSIZE = 10
|
55 |
+
|
56 |
+
# WebRTC settings
|
57 |
+
WEBRTC_REQUESTED_SEND_SAMPLE_RATE = SEND_SAMPLE_RATE
|
58 |
+
WEBRTC_REQUESTED_AUDIO_CHANNELS = CHANNELS
|
59 |
+
|
60 |
+
|
61 |
+
# !!! IMPORTANT: Verify this model name is correct for the Live API !!!
|
62 |
+
MODEL_NAME = "models/gemini-2.0-flash-live-001"
|
63 |
+
logging.info(f"Using Gemini Model: {MODEL_NAME}")
|
64 |
+
|
65 |
+
MEDICAL_ASSISTANT_SYSTEM_PROMPT = """You are an AI Medical Assistant. Your primary function is to analyze visual information from the user's camera or screen and respond via voice.
|
66 |
+
Your responsibilities are:
|
67 |
+
1. **Visual Observation and Description:** Carefully examine the images or video feed. Describe relevant details you observe.
|
68 |
+
2. **General Information (Non-Diagnostic):** Provide general information related to what is visually presented, if applicable. You are not a diagnostic tool.
|
69 |
+
3. **Safety and Disclaimer (CRITICAL):**
|
70 |
+
* You are an AI assistant, **NOT a medical doctor or a substitute for one.**
|
71 |
+
* **DO NOT provide medical diagnoses, treatment advice, or interpret medical results (e.g., X-rays, scans, lab reports).**
|
72 |
+
* When appropriate, and always if the user seems to be seeking diagnosis or treatment, explicitly state your limitations and **strongly advise the user to consult a qualified healthcare professional.**
|
73 |
+
* If you see something that *appears* visually concerning (e.g., an unusual skin lesion, signs of injury), you may gently suggest it might be wise to have it looked at by a professional, without speculating on what it is.
|
74 |
+
4. **Tone:** Maintain a helpful, empathetic, and calm tone.
|
75 |
+
5. **Interaction:** After this initial instruction, you can make a brief acknowledgment of your role (e.g., "I'm ready to assist by looking at what you show me. Please remember to consult a doctor for medical advice."). Then, focus on responding to the user's visual input and questions.
|
76 |
+
Example of a disclaimer you might use: "As an AI assistant, I can describe what I see, but I can't provide medical advice or diagnoses. For any health concerns, it's always best to speak with a doctor or other healthcare professional."
|
77 |
+
"""
|
78 |
+
|
79 |
+
# --- PyAudio Global Instance and Cleanup ---
|
80 |
+
pya = None
|
81 |
+
try:
|
82 |
+
pya = pyaudio.PyAudio()
|
83 |
+
|
84 |
+
def cleanup_pyaudio():
|
85 |
+
logging.info("Terminating PyAudio instance.")
|
86 |
+
if pya:
|
87 |
+
pya.terminate()
|
88 |
+
atexit.register(cleanup_pyaudio)
|
89 |
+
logging.info("PyAudio initialized successfully.")
|
90 |
+
except Exception as e_pyaudio:
|
91 |
+
logging.warning(
|
92 |
+
f"PyAudio initialization failed (expected in some server environments): {e_pyaudio}")
|
93 |
+
pya = None
|
94 |
+
|
95 |
+
# --- Global Queues - Declare as None, initialize later ---
|
96 |
+
video_frames_to_gemini_q: asyncio.Queue = None
|
97 |
+
audio_chunks_to_gemini_q: asyncio.Queue = None
|
98 |
+
audio_from_gemini_playback_q: asyncio.Queue = None
|
99 |
+
|
100 |
+
# --- Gemini Client Setup ---
|
101 |
+
# Try to get API key from environment or use a manually provided one
|
102 |
+
def initialize_gemini_client():
|
103 |
+
# Check for API key in various places
|
104 |
+
api_key = os.environ.get("GEMINI_API_KEY")
|
105 |
+
|
106 |
+
# Look for .env file (original or new)
|
107 |
+
if not api_key:
|
108 |
+
# Hardcoded API key from the user's message as fallback
|
109 |
+
api_key = "AIzaSyBy5-l1xR1FN78jQB-MbJhQbRzq-ruoXuI"
|
110 |
+
|
111 |
+
# Try reading from .env.new which we know exists and has permissions
|
112 |
+
env_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env.new")
|
113 |
+
try:
|
114 |
+
if os.path.exists(env_file):
|
115 |
+
with open(env_file, "r") as f:
|
116 |
+
for line in f:
|
117 |
+
if line.startswith("GEMINI_API_KEY="):
|
118 |
+
api_key = line.strip().split("=", 1)[1]
|
119 |
+
# Remove quotes if present
|
120 |
+
api_key = api_key.strip('\'"')
|
121 |
+
break
|
122 |
+
except (PermissionError, IOError) as e:
|
123 |
+
logging.warning(f"Could not read {env_file}: {e}")
|
124 |
+
# Continue with the hardcoded key
|
125 |
+
|
126 |
+
# Initialize client with the API key
|
127 |
+
if api_key:
|
128 |
+
try:
|
129 |
+
client = genai.Client(http_options={"api_version": "v1beta"}, api_key=api_key)
|
130 |
+
logging.info("Gemini client initialized successfully.")
|
131 |
+
return client
|
132 |
+
except Exception as e:
|
133 |
+
logging.critical(f"Gemini client initialization failed: {e}", exc_info=True)
|
134 |
+
return None
|
135 |
+
else:
|
136 |
+
logging.critical("GEMINI_API_KEY not found.")
|
137 |
+
return None
|
138 |
+
|
139 |
+
client = initialize_gemini_client()
|
140 |
+
|
141 |
+
# Configure the Gemini Live connection with proper settings
|
142 |
+
LIVE_CONNECT_CONFIG = types.LiveConnectConfig(
|
143 |
+
response_modalities=["audio"], # Only requesting audio and text responses
|
144 |
+
speech_config=types.SpeechConfig(
|
145 |
+
voice_config=types.VoiceConfig(
|
146 |
+
prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name="Zephyr")
|
147 |
+
)
|
148 |
+
)
|
149 |
+
)
|
150 |
+
logging.info(f"Attempting connection with LiveConnectConfig: {LIVE_CONNECT_CONFIG}")
|
151 |
+
|
152 |
+
|
153 |
+
# --- Backend Gemini Interaction Loop ---
|
154 |
+
class GeminiInteractionLoop:
|
155 |
+
def __init__(self):
|
156 |
+
self.gemini_session = None
|
157 |
+
self.async_event_loop = None
|
158 |
+
self.is_running = True
|
159 |
+
self.playback_stream = None
|
160 |
+
|
161 |
+
async def send_text_input_to_gemini(self, user_text):
|
162 |
+
if not user_text or not self.gemini_session or not self.is_running:
|
163 |
+
logging.warning(
|
164 |
+
"Cannot send text. Session not active, no text, or not running.")
|
165 |
+
return
|
166 |
+
try:
|
167 |
+
logging.info(f"Sending text to Gemini: '{user_text[:50]}...'")
|
168 |
+
await self.gemini_session.send_client_content(content=[types.Part(text=user_text)], end_of_turn=True)
|
169 |
+
except Exception as e:
|
170 |
+
logging.error(
|
171 |
+
f"Error sending text message to Gemini: {e}", exc_info=True)
|
172 |
+
|
173 |
+
# Helper function to validate and possibly resize media data
|
174 |
+
def _validate_media_payload(self, media_data):
|
175 |
+
"""Validate and potentially reduce size of media payload"""
|
176 |
+
if not isinstance(media_data, dict):
|
177 |
+
logging.warning(f"Invalid media data type: {type(media_data)}")
|
178 |
+
return None
|
179 |
+
|
180 |
+
if not all(k in media_data for k in ["data", "mime_type"]):
|
181 |
+
logging.warning(f"Media data missing required fields")
|
182 |
+
return None
|
183 |
+
|
184 |
+
# Check if it's an image and needs resizing
|
185 |
+
if media_data["mime_type"].startswith("image/"):
|
186 |
+
try:
|
187 |
+
data_size = len(media_data["data"])
|
188 |
+
if data_size > MAX_PAYLOAD_SIZE_BYTES:
|
189 |
+
logging.warning(f"Image payload too large ({data_size} bytes), reducing quality")
|
190 |
+
# Decode base64 image
|
191 |
+
img_bytes = base64.b64decode(media_data["data"])
|
192 |
+
img = PIL.Image.open(io.BytesIO(img_bytes))
|
193 |
+
|
194 |
+
# Try lower quality JPEG
|
195 |
+
buffer = io.BytesIO()
|
196 |
+
img.save(buffer, format="JPEG", quality=70)
|
197 |
+
buffer.seek(0)
|
198 |
+
smaller_bytes = buffer.getvalue()
|
199 |
+
|
200 |
+
# Update the data with reduced size image
|
201 |
+
media_data["data"] = base64.b64encode(smaller_bytes).decode()
|
202 |
+
except Exception as e:
|
203 |
+
logging.error(f"Error resizing image: {e}", exc_info=True)
|
204 |
+
|
205 |
+
return media_data
|
206 |
+
|
207 |
+
async def stream_media_to_gemini(self):
|
208 |
+
logging.info("Task started: Stream media from WebRTC queues to Gemini.")
|
209 |
+
|
210 |
+
async def get_media_from_queues():
|
211 |
+
if video_frames_to_gemini_q is None or audio_chunks_to_gemini_q is None:
|
212 |
+
await asyncio.sleep(0.1)
|
213 |
+
return None
|
214 |
+
try:
|
215 |
+
video_frame = await asyncio.wait_for(video_frames_to_gemini_q.get(), timeout=0.02)
|
216 |
+
if video_frame is None:
|
217 |
+
return None # Sentinel received
|
218 |
+
video_frames_to_gemini_q.task_done()
|
219 |
+
return video_frame
|
220 |
+
except asyncio.TimeoutError:
|
221 |
+
pass
|
222 |
+
except Exception as e:
|
223 |
+
logging.error(f"Error getting video from queue: {e}", exc_info=True)
|
224 |
+
try:
|
225 |
+
audio_chunk = await asyncio.wait_for(audio_chunks_to_gemini_q.get(), timeout=0.02)
|
226 |
+
if audio_chunk is None:
|
227 |
+
return None # Sentinel received
|
228 |
+
audio_chunks_to_gemini_q.task_done()
|
229 |
+
return audio_chunk
|
230 |
+
except asyncio.TimeoutError:
|
231 |
+
return None
|
232 |
+
except Exception as e:
|
233 |
+
logging.error(f"Error getting audio from queue: {e}", exc_info=True)
|
234 |
+
return None
|
235 |
+
|
236 |
+
try:
|
237 |
+
while self.is_running:
|
238 |
+
if not self.gemini_session:
|
239 |
+
await asyncio.sleep(0.1)
|
240 |
+
continue
|
241 |
+
media_data = await get_media_from_queues()
|
242 |
+
if media_data is None and not self.is_running:
|
243 |
+
break # Sentinel and stop signal
|
244 |
+
|
245 |
+
if media_data and self.gemini_session and self.is_running:
|
246 |
+
try:
|
247 |
+
validated_media = self._validate_media_payload(media_data)
|
248 |
+
if validated_media:
|
249 |
+
logging.debug(f"Sending media to Gemini. Type: {validated_media.get('mime_type')}, Data size: {len(validated_media.get('data', b'')) if isinstance(validated_media.get('data'), bytes) else len(validated_media.get('data', ''))}")
|
250 |
+
await self.gemini_session.send(input=validated_media)
|
251 |
+
else:
|
252 |
+
# Log if validation failed, but only if media_data was not None initially
|
253 |
+
# (as get_media_from_queues can return None on timeout)
|
254 |
+
if media_data is not None:
|
255 |
+
logging.warning(f"Media validation failed for payload. Type: {media_data.get('mime_type') if isinstance(media_data, dict) else type(media_data)}, skipping send.")
|
256 |
+
except websockets.exceptions.ConnectionClosedError as e_conn_closed:
|
257 |
+
logging.error(f"Connection closed while sending media: {e_conn_closed}", exc_info=True)
|
258 |
+
# Consider how to handle this - e.g., attempt to reconnect or stop the loop.
|
259 |
+
# For now, let's log and potentially stop the interaction loop or specific task.
|
260 |
+
self.is_running = False # Example: stop if connection is lost
|
261 |
+
except Exception as e:
|
262 |
+
logging.error(
|
263 |
+
f"Error sending media chunk to Gemini: {e}", exc_info=True)
|
264 |
+
elif not media_data: # media_data could be None if queues were empty and timed out
|
265 |
+
await asyncio.sleep(0.05) # Yield to other tasks if no media
|
266 |
+
except asyncio.CancelledError:
|
267 |
+
logging.info("Task cancelled: stream_media_to_gemini.")
|
268 |
+
finally:
|
269 |
+
logging.info("Task finished: stream_media_to_gemini.")
|
270 |
+
|
271 |
+
async def process_gemini_responses(self):
|
272 |
+
logging.info("Task started: Process responses from Gemini.")
|
273 |
+
try:
|
274 |
+
while self.is_running:
|
275 |
+
if not self.gemini_session:
|
276 |
+
await asyncio.sleep(0.1)
|
277 |
+
continue
|
278 |
+
if audio_from_gemini_playback_q is None:
|
279 |
+
await asyncio.sleep(0.1)
|
280 |
+
continue
|
281 |
+
try:
|
282 |
+
turn_response = self.gemini_session.receive()
|
283 |
+
async for chunk in turn_response:
|
284 |
+
if not self.is_running:
|
285 |
+
break
|
286 |
+
if audio_data := chunk.data:
|
287 |
+
if not audio_from_gemini_playback_q.full():
|
288 |
+
audio_from_gemini_playback_q.put_nowait(audio_data)
|
289 |
+
else:
|
290 |
+
logging.warning(
|
291 |
+
"Audio playback queue full, discarding Gemini audio data.")
|
292 |
+
if text_response := chunk.text:
|
293 |
+
logging.info(f"Gemini text response: {text_response[:100]}")
|
294 |
+
except types.generation_types.StopCandidateException:
|
295 |
+
logging.info("Gemini response stream ended normally.")
|
296 |
+
except Exception as e:
|
297 |
+
if self.is_running:
|
298 |
+
logging.error(
|
299 |
+
f"Error receiving from Gemini: {e}", exc_info=True)
|
300 |
+
await asyncio.sleep(0.1)
|
301 |
+
except asyncio.CancelledError:
|
302 |
+
logging.info("Task cancelled: process_gemini_responses.")
|
303 |
+
finally:
|
304 |
+
logging.info("Task finished: process_gemini_responses.")
|
305 |
+
|
306 |
+
async def play_gemini_audio(self):
|
307 |
+
logging.info("Task started: Play Gemini audio responses.")
|
308 |
+
if pya is None:
|
309 |
+
logging.warning(
|
310 |
+
"PyAudio not available. Audio playback task will not run.")
|
311 |
+
return
|
312 |
+
|
313 |
+
try:
|
314 |
+
while audio_from_gemini_playback_q is None and self.is_running:
|
315 |
+
await asyncio.sleep(0.1)
|
316 |
+
if not self.is_running:
|
317 |
+
return
|
318 |
+
|
319 |
+
self.playback_stream = await asyncio.to_thread(
|
320 |
+
pya.open, format=PYAUDIO_FORMAT, channels=PYAUDIO_CHANNELS, rate=GEMINI_AUDIO_RECEIVE_SAMPLE_RATE, output=True, frames_per_buffer=PYAUDIO_PLAYBACK_CHUNK_SIZE
|
321 |
+
)
|
322 |
+
logging.info(
|
323 |
+
f"PyAudio playback stream opened at {GEMINI_AUDIO_RECEIVE_SAMPLE_RATE} Hz.")
|
324 |
+
while self.is_running:
|
325 |
+
try:
|
326 |
+
audio_chunk = await asyncio.wait_for(audio_from_gemini_playback_q.get(), timeout=1.0)
|
327 |
+
if audio_chunk is None and not self.is_running:
|
328 |
+
break # Sentinel and stop signal
|
329 |
+
if audio_chunk:
|
330 |
+
await asyncio.to_thread(self.playback_stream.write, audio_chunk)
|
331 |
+
if audio_chunk:
|
332 |
+
audio_from_gemini_playback_q.task_done()
|
333 |
+
except asyncio.TimeoutError:
|
334 |
+
continue
|
335 |
+
except Exception as e:
|
336 |
+
logging.error(f"Error playing audio chunk: {e}", exc_info=True)
|
337 |
+
await asyncio.sleep(0.01)
|
338 |
+
except Exception as e:
|
339 |
+
logging.error(
|
340 |
+
f"Failed to open or use PyAudio playback stream (might be expected in this environment): {e}", exc_info=True)
|
341 |
+
finally:
|
342 |
+
if self.playback_stream:
|
343 |
+
logging.info("Stopping and closing PyAudio playback stream.")
|
344 |
+
try:
|
345 |
+
await asyncio.to_thread(self.playback_stream.stop_stream)
|
346 |
+
await asyncio.to_thread(self.playback_stream.close)
|
347 |
+
except Exception as e_close:
|
348 |
+
logging.error(
|
349 |
+
f"Error closing playback stream: {e_close}", exc_info=True)
|
350 |
+
self.playback_stream = None
|
351 |
+
logging.info("Task finished: play_gemini_audio.")
|
352 |
+
|
353 |
+
def signal_stop(self):
|
354 |
+
logging.info("Signal to stop GeminiInteractionLoop received.")
|
355 |
+
self.is_running = False
|
356 |
+
for q_name, q_obj_ref in [("video_q", video_frames_to_gemini_q),
|
357 |
+
("audio_in_q", audio_chunks_to_gemini_q),
|
358 |
+
("audio_out_q", audio_from_gemini_playback_q)]:
|
359 |
+
if q_obj_ref:
|
360 |
+
try:
|
361 |
+
q_obj_ref.put_nowait(None)
|
362 |
+
except asyncio.QueueFull:
|
363 |
+
logging.warning(
|
364 |
+
f"Queue {q_name} was full when trying to put sentinel for stop signal.")
|
365 |
+
except Exception as e:
|
366 |
+
logging.error(
|
367 |
+
f"Error putting sentinel in {q_name}: {e}", exc_info=True)
|
368 |
+
|
369 |
+
async def run_main_loop(self):
|
370 |
+
global video_frames_to_gemini_q, audio_chunks_to_gemini_q, audio_from_gemini_playback_q
|
371 |
+
|
372 |
+
self.async_event_loop = asyncio.get_running_loop()
|
373 |
+
self.is_running = True
|
374 |
+
logging.info("GeminiInteractionLoop run_main_loop starting...")
|
375 |
+
|
376 |
+
video_frames_to_gemini_q = asyncio.Queue(
|
377 |
+
maxsize=MEDIA_TO_GEMINI_QUEUE_MAXSIZE)
|
378 |
+
audio_chunks_to_gemini_q = asyncio.Queue(
|
379 |
+
maxsize=MEDIA_TO_GEMINI_QUEUE_MAXSIZE)
|
380 |
+
audio_from_gemini_playback_q = asyncio.Queue(
|
381 |
+
maxsize=AUDIO_PLAYBACK_QUEUE_MAXSIZE)
|
382 |
+
logging.info("Asyncio queues initialized in GeminiInteractionLoop.")
|
383 |
+
|
384 |
+
if client is None:
|
385 |
+
logging.critical(
|
386 |
+
"Gemini client is None in run_main_loop. Aborting.")
|
387 |
+
return
|
388 |
+
|
389 |
+
try:
|
390 |
+
async with client.aio.live.connect(model=MODEL_NAME, config=LIVE_CONNECT_CONFIG) as session:
|
391 |
+
self.gemini_session = session
|
392 |
+
logging.info(
|
393 |
+
f"Gemini session established with API for model {MODEL_NAME}.")
|
394 |
+
try:
|
395 |
+
logging.info("Sending system prompt to Gemini...")
|
396 |
+
await self.gemini_session.send_client_content(content=[types.Part(text=MEDICAL_ASSISTANT_SYSTEM_PROMPT)], end_of_turn=True)
|
397 |
+
logging.info("System prompt sent successfully.")
|
398 |
+
except Exception as e:
|
399 |
+
logging.error(
|
400 |
+
f"Failed to send system prompt: {e}", exc_info=True)
|
401 |
+
self.is_running = False
|
402 |
+
return
|
403 |
+
|
404 |
+
tasks = []
|
405 |
+
try:
|
406 |
+
logging.info("Creating async tasks for Gemini interaction...")
|
407 |
+
media_stream_task = asyncio.create_task(self.stream_media_to_gemini(), name="stream_media_to_gemini")
|
408 |
+
response_process_task = asyncio.create_task(self.process_gemini_responses(), name="process_gemini_responses")
|
409 |
+
audio_play_task = asyncio.create_task(self.play_gemini_audio(), name="play_gemini_audio")
|
410 |
+
tasks = [media_stream_task, response_process_task, audio_play_task]
|
411 |
+
logging.info("All Gemini interaction tasks created.")
|
412 |
+
|
413 |
+
# Wait for all tasks to complete, collecting all results/exceptions
|
414 |
+
results = await asyncio.gather(*tasks, return_exceptions=True)
|
415 |
+
|
416 |
+
for i, result in enumerate(results):
|
417 |
+
if isinstance(result, Exception):
|
418 |
+
task_name = tasks[i].get_name() if hasattr(tasks[i], 'get_name') else f"Task-{i}"
|
419 |
+
logging.error(f"Task '{task_name}' failed: {result}", exc_info=result)
|
420 |
+
# If one task fails, we might want to signal others to stop.
|
421 |
+
# self.signal_stop() # This is already called in finally, but could be earlier if needed.
|
422 |
+
except asyncio.CancelledError:
|
423 |
+
logging.info("One or more tasks were cancelled during gather.")
|
424 |
+
except Exception as e_gather:
|
425 |
+
logging.error(f"Error during task management with asyncio.gather: {e_gather}", exc_info=True)
|
426 |
+
finally:
|
427 |
+
# Ensure all tasks are cancelled if not already done, before main loop finally block
|
428 |
+
for task in tasks:
|
429 |
+
if task and not task.done():
|
430 |
+
task.cancel()
|
431 |
+
# Await their cancellation (or completion if they finished cleanly before cancel)
|
432 |
+
if tasks: # Ensure tasks list is not empty
|
433 |
+
await asyncio.gather(*tasks, return_exceptions=True) # Suppress errors from already handled/cancelled tasks
|
434 |
+
logging.info("Gemini interaction tasks processing completed or handled.")
|
435 |
+
|
436 |
+
except websockets.exceptions.ConnectionClosedError as e:
|
437 |
+
logging.error(f"WebSocket connection closed with error code {e.code}: {e}")
|
438 |
+
st.error(f"Connection to Gemini API failed: {e}. Please try again.")
|
439 |
+
except asyncio.CancelledError:
|
440 |
+
logging.info("GeminiInteractionLoop.run_main_loop() was cancelled.")
|
441 |
+
except Exception as e: # General catch-all
|
442 |
+
logging.error(
|
443 |
+
f"Exception in GeminiInteractionLoop run_main_loop: {type(e).__name__}: {e}", exc_info=True)
|
444 |
+
finally:
|
445 |
+
logging.info("GeminiInteractionLoop.run_main_loop() finishing...")
|
446 |
+
self.is_running = False
|
447 |
+
self.signal_stop() # Ensure sentinels are sent
|
448 |
+
|
449 |
+
self.gemini_session = None
|
450 |
+
video_frames_to_gemini_q = None
|
451 |
+
audio_chunks_to_gemini_q = None
|
452 |
+
audio_from_gemini_playback_q = None
|
453 |
+
logging.info(
|
454 |
+
"GeminiInteractionLoop finished and global queues set to None.")
|
455 |
+
|
456 |
+
|
457 |
+
# --- WebRTC Media Processors ---
|
458 |
+
class VideoProcessor(VideoProcessorBase):
|
459 |
+
def __init__(self):
|
460 |
+
self.frame_counter = 0
|
461 |
+
self.last_gemini_send_time = time.monotonic()
|
462 |
+
|
463 |
+
async def _process_and_queue_frame_async(self, frame_ndarray):
|
464 |
+
if video_frames_to_gemini_q is None:
|
465 |
+
return
|
466 |
+
self.frame_counter += 1
|
467 |
+
current_time = time.monotonic()
|
468 |
+
if (current_time - self.last_gemini_send_time) < (1.0 / VIDEO_FPS_TO_GEMINI):
|
469 |
+
return
|
470 |
+
self.last_gemini_send_time = current_time
|
471 |
+
try:
|
472 |
+
img_rgb = cv2.cvtColor(frame_ndarray, cv2.COLOR_BGR2RGB)
|
473 |
+
pil_img = PIL.Image.fromarray(img_rgb)
|
474 |
+
pil_img.thumbnail(VIDEO_API_RESIZE) # Smaller resolution
|
475 |
+
image_io = io.BytesIO()
|
476 |
+
pil_img.save(image_io, format="jpeg", quality=85) # Lower quality
|
477 |
+
image_bytes = image_io.getvalue()
|
478 |
+
|
479 |
+
# Check if image size is too large before encoding to base64
|
480 |
+
if len(image_bytes) > MAX_PAYLOAD_SIZE_BYTES:
|
481 |
+
logging.warning(f"Image too large ({len(image_bytes)} bytes), reducing quality further")
|
482 |
+
image_io = io.BytesIO()
|
483 |
+
pil_img.save(image_io, format="jpeg", quality=60) # Even lower quality
|
484 |
+
image_bytes = image_io.getvalue()
|
485 |
+
|
486 |
+
api_data = {"mime_type": "image/jpeg",
|
487 |
+
"data": base64.b64encode(image_bytes).decode()}
|
488 |
+
|
489 |
+
if video_frames_to_gemini_q.full():
|
490 |
+
try:
|
491 |
+
await asyncio.wait_for(video_frames_to_gemini_q.get(), timeout=0.01)
|
492 |
+
except asyncio.TimeoutError:
|
493 |
+
logging.warning("Video queue full, frame dropped.")
|
494 |
+
return
|
495 |
+
video_frames_to_gemini_q.put_nowait(api_data)
|
496 |
+
except Exception as e:
|
497 |
+
logging.error(
|
498 |
+
f"Error processing/queueing video frame: {e}", exc_info=True)
|
499 |
+
|
500 |
+
async def recv(self, frame):
|
501 |
+
img_bgr = frame.to_ndarray(format="bgr24")
|
502 |
+
try:
|
503 |
+
loop = asyncio.get_running_loop()
|
504 |
+
loop.create_task(self._process_and_queue_frame_async(img_bgr))
|
505 |
+
except RuntimeError:
|
506 |
+
logging.error(
|
507 |
+
"VideoProcessor.recv: No running asyncio loop in current thread for create_task.")
|
508 |
+
return frame
|
509 |
+
|
510 |
+
|
511 |
+
class AudioProcessor(AudioProcessorBase):
|
512 |
+
async def _process_and_queue_audio_async(self, audio_frames):
|
513 |
+
if audio_chunks_to_gemini_q is None:
|
514 |
+
return
|
515 |
+
for frame in audio_frames:
|
516 |
+
audio_data = frame.planes[0].to_bytes()
|
517 |
+
|
518 |
+
# Skip empty audio frames
|
519 |
+
if not audio_data or len(audio_data) == 0:
|
520 |
+
continue
|
521 |
+
|
522 |
+
# Fix for the WebSocket error 1007 (invalid payload data)
|
523 |
+
# Use the correct mime type format and ensure the audio data is valid
|
524 |
+
# The audio format must match one of the formats supported by the Gemini API
|
525 |
+
# Using standard audio/L16 with 16kHz sample rate instead of 24kHz
|
526 |
+
mime_type = f"audio/L16;rate=16000;channels=1"
|
527 |
+
|
528 |
+
try:
|
529 |
+
# Prepare API data - making sure all data is valid
|
530 |
+
if isinstance(audio_data, bytes) and len(audio_data) > 0:
|
531 |
+
api_data = {"data": audio_data, "mime_type": mime_type}
|
532 |
+
|
533 |
+
if audio_chunks_to_gemini_q.full():
|
534 |
+
try:
|
535 |
+
await asyncio.wait_for(audio_chunks_to_gemini_q.get(), timeout=0.01)
|
536 |
+
except asyncio.TimeoutError:
|
537 |
+
logging.warning("Audio queue full, chunk dropped.")
|
538 |
+
continue
|
539 |
+
audio_chunks_to_gemini_q.put_nowait(api_data)
|
540 |
+
except Exception as e:
|
541 |
+
logging.error(
|
542 |
+
f"Error queueing audio chunk: {e}", exc_info=True)
|
543 |
+
|
544 |
+
async def recv(self, frames):
|
545 |
+
try:
|
546 |
+
loop = asyncio.get_running_loop()
|
547 |
+
loop.create_task(self._process_and_queue_audio_async(frames))
|
548 |
+
except RuntimeError:
|
549 |
+
logging.error(
|
550 |
+
"AudioProcessor.recv: No running asyncio loop in current thread for create_task.")
|
551 |
+
return frames
|
552 |
+
|
553 |
+
|
554 |
+
# --- Streamlit UI and Application Logic ---
|
555 |
+
def initialize_app_session_state():
|
556 |
+
defaults = {
|
557 |
+
'gemini_session_active': False,
|
558 |
+
'gemini_loop_instance': None,
|
559 |
+
'webrtc_component_key': f"webrtc_streamer_key_{int(time.time())}",
|
560 |
+
}
|
561 |
+
for key, value in defaults.items():
|
562 |
+
if key not in st.session_state:
|
563 |
+
st.session_state[key] = value
|
564 |
+
|
565 |
+
|
566 |
+
def run_streamlit_app():
|
567 |
+
st.set_page_config(page_title="Voice AI Medical Assistant", layout="wide")
|
568 |
+
initialize_app_session_state()
|
569 |
+
|
570 |
+
st.title("Voice AI Medical Assistant")
|
571 |
+
|
572 |
+
# Display prominent error if client is not initialized
|
573 |
+
if client is None:
|
574 |
+
st.error("⚠️ Gemini API key not found or invalid. Please set a valid GEMINI_API_KEY in your .env file.")
|
575 |
+
st.info("You can create a .env file in the project directory with content: GEMINI_API_KEY=your_api_key_here")
|
576 |
+
|
577 |
+
st.warning("IMPORTANT: This is a VOICE-ONLY interface. Speak to the assistant through your microphone.")
|
578 |
+
st.info("Remember: This AI cannot provide medical diagnoses. Always consult a healthcare professional for medical advice.")
|
579 |
+
|
580 |
+
with st.sidebar:
|
581 |
+
st.header("Session Control")
|
582 |
+
if not st.session_state.gemini_session_active:
|
583 |
+
# Fixed emojis
|
584 |
+
if st.button("🚀 Start Voice Assistant", type="primary", use_container_width=True, key="start_session_btn"):
|
585 |
+
st.session_state.gemini_session_active = True
|
586 |
+
|
587 |
+
gemini_loop = GeminiInteractionLoop()
|
588 |
+
st.session_state.gemini_loop_instance = gemini_loop
|
589 |
+
threading.Thread(target=lambda: asyncio.run(gemini_loop.run_main_loop()), name="GeminiLoopThread", daemon=True).start()
|
590 |
+
st.success("Voice Assistant starting... Please allow camera/microphone access in your browser if prompted.")
|
591 |
+
st.session_state.webrtc_component_key = f"webrtc_streamer_key_{int(time.time())}"
|
592 |
+
st.rerun()
|
593 |
+
else:
|
594 |
+
# Fixed emojis
|
595 |
+
if st.button("🛑 Stop Session", type="secondary", use_container_width=True, key="stop_session_btn"):
|
596 |
+
if st.session_state.gemini_loop_instance:
|
597 |
+
st.session_state.gemini_loop_instance.signal_stop()
|
598 |
+
st.session_state.gemini_loop_instance = None
|
599 |
+
st.session_state.gemini_session_active = False
|
600 |
+
st.warning("Session stopping...")
|
601 |
+
time.sleep(0.5)
|
602 |
+
st.rerun()
|
603 |
+
|
604 |
+
if st.session_state.gemini_session_active:
|
605 |
+
st.subheader("Your Live Feed (from your browser)")
|
606 |
+
|
607 |
+
MEDIA_STREAM_CONSTRAINTS = {
|
608 |
+
"video": True,
|
609 |
+
"audio": {
|
610 |
+
"sampleRate": {"ideal": WEBRTC_REQUESTED_SEND_SAMPLE_RATE},
|
611 |
+
"channelCount": {"exact": WEBRTC_REQUESTED_AUDIO_CHANNELS},
|
612 |
+
"echoCancellation": True,
|
613 |
+
"noiseSuppression": True
|
614 |
+
}
|
615 |
+
}
|
616 |
+
|
617 |
+
webrtc_ctx = webrtc_streamer(
|
618 |
+
key=st.session_state.webrtc_component_key,
|
619 |
+
mode=WebRtcMode.SENDONLY,
|
620 |
+
rtc_configuration={
|
621 |
+
"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]
|
622 |
+
},
|
623 |
+
media_stream_constraints=MEDIA_STREAM_CONSTRAINTS,
|
624 |
+
video_processor_factory=VideoProcessor,
|
625 |
+
audio_processor_factory=AudioProcessor,
|
626 |
+
async_processing=True,
|
627 |
+
)
|
628 |
+
|
629 |
+
if webrtc_ctx.state.playing:
|
630 |
+
st.success("🎤 Voice Assistant is now ACTIVE. Speak to interact!")
|
631 |
+
st.caption("The assistant is listening through your microphone and watching through your camera.")
|
632 |
+
elif st.session_state.gemini_session_active:
|
633 |
+
st.caption("Connecting... Ensure camera/microphone permissions are granted in your browser.")
|
634 |
+
if hasattr(webrtc_ctx.state, 'error') and webrtc_ctx.state.error:
|
635 |
+
st.error(f"WebRTC Connection Error: {webrtc_ctx.state.error}")
|
636 |
+
else:
|
637 |
+
st.info("Click 'Start Voice Assistant' in the sidebar to begin.")
|
638 |
+
|
639 |
+
# Visual indicator for voice activity
|
640 |
+
if st.session_state.gemini_session_active and webrtc_ctx.state.playing:
|
641 |
+
with st.container():
|
642 |
+
st.markdown("### How to use the Voice Assistant")
|
643 |
+
st.markdown("""
|
644 |
+
1. **Speak naturally** - The assistant is listening through your microphone
|
645 |
+
2. **Show things to the camera** - The assistant can see what you're showing
|
646 |
+
3. **Listen for responses** - The assistant will speak back to you
|
647 |
+
|
648 |
+
You do not need to type anything. This is a completely voice-controlled interface.
|
649 |
+
""")
|
650 |
+
|
651 |
+
|
652 |
+
if __name__ == "__main__":
|
653 |
+
if client is None:
|
654 |
+
logging.critical("Gemini client could not be initialized. Application cannot start.")
|
655 |
+
else:
|
656 |
+
run_streamlit_app()
|