noumanjavaid commited on
Commit
67ea58d
·
verified ·
1 Parent(s): 21e17e5

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +656 -0
app.py ADDED
@@ -0,0 +1,656 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ import streamlit as st
3
+ import os
4
+ import asyncio
5
+ import base64
6
+ import io
7
+ import threading
8
+ import traceback
9
+ import atexit
10
+ import time
11
+ import logging
12
+ from dotenv import load_dotenv
13
+
14
+ import cv2
15
+ import pyaudio
16
+ import PIL.Image
17
+
18
+ # Import websockets for explicit exception handling
19
+ import websockets.exceptions
20
+
21
+ from google import genai
22
+ from google.genai import types
23
+
24
+ from streamlit_webrtc import (
25
+ webrtc_streamer,
26
+ WebRtcMode,
27
+ AudioProcessorBase,
28
+ VideoProcessorBase,
29
+ )
30
+
31
+ load_dotenv()
32
+
33
+
34
+ # Audio configuration - fix audio format issues
35
+ FORMAT = pyaudio.paInt16
36
+ CHANNELS = 1
37
+ SEND_SAMPLE_RATE = 16000 # Changed to match mime_type for consistency
38
+ RECEIVE_SAMPLE_RATE = 16000 # Changed from 24000 to 16000 to match send rate
39
+ CHUNK_SIZE = 1024
40
+
41
+ # Map PyAudio format to a more descriptive name for clarity.
42
+ PYAUDIO_FORMAT = FORMAT # pyaudio.paInt16
43
+ PYAUDIO_CHANNELS = CHANNELS
44
+ PYAUDIO_PLAYBACK_CHUNK_SIZE = CHUNK_SIZE
45
+ GEMINI_AUDIO_RECEIVE_SAMPLE_RATE = RECEIVE_SAMPLE_RATE
46
+
47
+ # Video configuration
48
+ VIDEO_FPS_TO_GEMINI = 1 # Reduced from 2 to lower bandwidth
49
+ VIDEO_API_RESIZE = (512, 512) # Reduced from 1024x1024 to lower payload size
50
+ MAX_PAYLOAD_SIZE_BYTES = 60000 # Just under 64KB WebSocket limit
51
+
52
+ # Queue sizes
53
+ MEDIA_TO_GEMINI_QUEUE_MAXSIZE = 10
54
+ AUDIO_PLAYBACK_QUEUE_MAXSIZE = 10
55
+
56
+ # WebRTC settings
57
+ WEBRTC_REQUESTED_SEND_SAMPLE_RATE = SEND_SAMPLE_RATE
58
+ WEBRTC_REQUESTED_AUDIO_CHANNELS = CHANNELS
59
+
60
+
61
+ # !!! IMPORTANT: Verify this model name is correct for the Live API !!!
62
+ MODEL_NAME = "models/gemini-2.0-flash-live-001"
63
+ logging.info(f"Using Gemini Model: {MODEL_NAME}")
64
+
65
+ MEDICAL_ASSISTANT_SYSTEM_PROMPT = """You are an AI Medical Assistant. Your primary function is to analyze visual information from the user's camera or screen and respond via voice.
66
+ Your responsibilities are:
67
+ 1. **Visual Observation and Description:** Carefully examine the images or video feed. Describe relevant details you observe.
68
+ 2. **General Information (Non-Diagnostic):** Provide general information related to what is visually presented, if applicable. You are not a diagnostic tool.
69
+ 3. **Safety and Disclaimer (CRITICAL):**
70
+ * You are an AI assistant, **NOT a medical doctor or a substitute for one.**
71
+ * **DO NOT provide medical diagnoses, treatment advice, or interpret medical results (e.g., X-rays, scans, lab reports).**
72
+ * When appropriate, and always if the user seems to be seeking diagnosis or treatment, explicitly state your limitations and **strongly advise the user to consult a qualified healthcare professional.**
73
+ * If you see something that *appears* visually concerning (e.g., an unusual skin lesion, signs of injury), you may gently suggest it might be wise to have it looked at by a professional, without speculating on what it is.
74
+ 4. **Tone:** Maintain a helpful, empathetic, and calm tone.
75
+ 5. **Interaction:** After this initial instruction, you can make a brief acknowledgment of your role (e.g., "I'm ready to assist by looking at what you show me. Please remember to consult a doctor for medical advice."). Then, focus on responding to the user's visual input and questions.
76
+ Example of a disclaimer you might use: "As an AI assistant, I can describe what I see, but I can't provide medical advice or diagnoses. For any health concerns, it's always best to speak with a doctor or other healthcare professional."
77
+ """
78
+
79
+ # --- PyAudio Global Instance and Cleanup ---
80
+ pya = None
81
+ try:
82
+ pya = pyaudio.PyAudio()
83
+
84
+ def cleanup_pyaudio():
85
+ logging.info("Terminating PyAudio instance.")
86
+ if pya:
87
+ pya.terminate()
88
+ atexit.register(cleanup_pyaudio)
89
+ logging.info("PyAudio initialized successfully.")
90
+ except Exception as e_pyaudio:
91
+ logging.warning(
92
+ f"PyAudio initialization failed (expected in some server environments): {e_pyaudio}")
93
+ pya = None
94
+
95
+ # --- Global Queues - Declare as None, initialize later ---
96
+ video_frames_to_gemini_q: asyncio.Queue = None
97
+ audio_chunks_to_gemini_q: asyncio.Queue = None
98
+ audio_from_gemini_playback_q: asyncio.Queue = None
99
+
100
+ # --- Gemini Client Setup ---
101
+ # Try to get API key from environment or use a manually provided one
102
+ def initialize_gemini_client():
103
+ # Check for API key in various places
104
+ api_key = os.environ.get("GEMINI_API_KEY")
105
+
106
+ # Look for .env file (original or new)
107
+ if not api_key:
108
+ # Hardcoded API key from the user's message as fallback
109
+ api_key = "AIzaSyBy5-l1xR1FN78jQB-MbJhQbRzq-ruoXuI"
110
+
111
+ # Try reading from .env.new which we know exists and has permissions
112
+ env_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env.new")
113
+ try:
114
+ if os.path.exists(env_file):
115
+ with open(env_file, "r") as f:
116
+ for line in f:
117
+ if line.startswith("GEMINI_API_KEY="):
118
+ api_key = line.strip().split("=", 1)[1]
119
+ # Remove quotes if present
120
+ api_key = api_key.strip('\'"')
121
+ break
122
+ except (PermissionError, IOError) as e:
123
+ logging.warning(f"Could not read {env_file}: {e}")
124
+ # Continue with the hardcoded key
125
+
126
+ # Initialize client with the API key
127
+ if api_key:
128
+ try:
129
+ client = genai.Client(http_options={"api_version": "v1beta"}, api_key=api_key)
130
+ logging.info("Gemini client initialized successfully.")
131
+ return client
132
+ except Exception as e:
133
+ logging.critical(f"Gemini client initialization failed: {e}", exc_info=True)
134
+ return None
135
+ else:
136
+ logging.critical("GEMINI_API_KEY not found.")
137
+ return None
138
+
139
+ client = initialize_gemini_client()
140
+
141
+ # Configure the Gemini Live connection with proper settings
142
+ LIVE_CONNECT_CONFIG = types.LiveConnectConfig(
143
+ response_modalities=["audio"], # Only requesting audio and text responses
144
+ speech_config=types.SpeechConfig(
145
+ voice_config=types.VoiceConfig(
146
+ prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name="Zephyr")
147
+ )
148
+ )
149
+ )
150
+ logging.info(f"Attempting connection with LiveConnectConfig: {LIVE_CONNECT_CONFIG}")
151
+
152
+
153
+ # --- Backend Gemini Interaction Loop ---
154
+ class GeminiInteractionLoop:
155
+ def __init__(self):
156
+ self.gemini_session = None
157
+ self.async_event_loop = None
158
+ self.is_running = True
159
+ self.playback_stream = None
160
+
161
+ async def send_text_input_to_gemini(self, user_text):
162
+ if not user_text or not self.gemini_session or not self.is_running:
163
+ logging.warning(
164
+ "Cannot send text. Session not active, no text, or not running.")
165
+ return
166
+ try:
167
+ logging.info(f"Sending text to Gemini: '{user_text[:50]}...'")
168
+ await self.gemini_session.send_client_content(content=[types.Part(text=user_text)], end_of_turn=True)
169
+ except Exception as e:
170
+ logging.error(
171
+ f"Error sending text message to Gemini: {e}", exc_info=True)
172
+
173
+ # Helper function to validate and possibly resize media data
174
+ def _validate_media_payload(self, media_data):
175
+ """Validate and potentially reduce size of media payload"""
176
+ if not isinstance(media_data, dict):
177
+ logging.warning(f"Invalid media data type: {type(media_data)}")
178
+ return None
179
+
180
+ if not all(k in media_data for k in ["data", "mime_type"]):
181
+ logging.warning(f"Media data missing required fields")
182
+ return None
183
+
184
+ # Check if it's an image and needs resizing
185
+ if media_data["mime_type"].startswith("image/"):
186
+ try:
187
+ data_size = len(media_data["data"])
188
+ if data_size > MAX_PAYLOAD_SIZE_BYTES:
189
+ logging.warning(f"Image payload too large ({data_size} bytes), reducing quality")
190
+ # Decode base64 image
191
+ img_bytes = base64.b64decode(media_data["data"])
192
+ img = PIL.Image.open(io.BytesIO(img_bytes))
193
+
194
+ # Try lower quality JPEG
195
+ buffer = io.BytesIO()
196
+ img.save(buffer, format="JPEG", quality=70)
197
+ buffer.seek(0)
198
+ smaller_bytes = buffer.getvalue()
199
+
200
+ # Update the data with reduced size image
201
+ media_data["data"] = base64.b64encode(smaller_bytes).decode()
202
+ except Exception as e:
203
+ logging.error(f"Error resizing image: {e}", exc_info=True)
204
+
205
+ return media_data
206
+
207
+ async def stream_media_to_gemini(self):
208
+ logging.info("Task started: Stream media from WebRTC queues to Gemini.")
209
+
210
+ async def get_media_from_queues():
211
+ if video_frames_to_gemini_q is None or audio_chunks_to_gemini_q is None:
212
+ await asyncio.sleep(0.1)
213
+ return None
214
+ try:
215
+ video_frame = await asyncio.wait_for(video_frames_to_gemini_q.get(), timeout=0.02)
216
+ if video_frame is None:
217
+ return None # Sentinel received
218
+ video_frames_to_gemini_q.task_done()
219
+ return video_frame
220
+ except asyncio.TimeoutError:
221
+ pass
222
+ except Exception as e:
223
+ logging.error(f"Error getting video from queue: {e}", exc_info=True)
224
+ try:
225
+ audio_chunk = await asyncio.wait_for(audio_chunks_to_gemini_q.get(), timeout=0.02)
226
+ if audio_chunk is None:
227
+ return None # Sentinel received
228
+ audio_chunks_to_gemini_q.task_done()
229
+ return audio_chunk
230
+ except asyncio.TimeoutError:
231
+ return None
232
+ except Exception as e:
233
+ logging.error(f"Error getting audio from queue: {e}", exc_info=True)
234
+ return None
235
+
236
+ try:
237
+ while self.is_running:
238
+ if not self.gemini_session:
239
+ await asyncio.sleep(0.1)
240
+ continue
241
+ media_data = await get_media_from_queues()
242
+ if media_data is None and not self.is_running:
243
+ break # Sentinel and stop signal
244
+
245
+ if media_data and self.gemini_session and self.is_running:
246
+ try:
247
+ validated_media = self._validate_media_payload(media_data)
248
+ if validated_media:
249
+ logging.debug(f"Sending media to Gemini. Type: {validated_media.get('mime_type')}, Data size: {len(validated_media.get('data', b'')) if isinstance(validated_media.get('data'), bytes) else len(validated_media.get('data', ''))}")
250
+ await self.gemini_session.send(input=validated_media)
251
+ else:
252
+ # Log if validation failed, but only if media_data was not None initially
253
+ # (as get_media_from_queues can return None on timeout)
254
+ if media_data is not None:
255
+ logging.warning(f"Media validation failed for payload. Type: {media_data.get('mime_type') if isinstance(media_data, dict) else type(media_data)}, skipping send.")
256
+ except websockets.exceptions.ConnectionClosedError as e_conn_closed:
257
+ logging.error(f"Connection closed while sending media: {e_conn_closed}", exc_info=True)
258
+ # Consider how to handle this - e.g., attempt to reconnect or stop the loop.
259
+ # For now, let's log and potentially stop the interaction loop or specific task.
260
+ self.is_running = False # Example: stop if connection is lost
261
+ except Exception as e:
262
+ logging.error(
263
+ f"Error sending media chunk to Gemini: {e}", exc_info=True)
264
+ elif not media_data: # media_data could be None if queues were empty and timed out
265
+ await asyncio.sleep(0.05) # Yield to other tasks if no media
266
+ except asyncio.CancelledError:
267
+ logging.info("Task cancelled: stream_media_to_gemini.")
268
+ finally:
269
+ logging.info("Task finished: stream_media_to_gemini.")
270
+
271
+ async def process_gemini_responses(self):
272
+ logging.info("Task started: Process responses from Gemini.")
273
+ try:
274
+ while self.is_running:
275
+ if not self.gemini_session:
276
+ await asyncio.sleep(0.1)
277
+ continue
278
+ if audio_from_gemini_playback_q is None:
279
+ await asyncio.sleep(0.1)
280
+ continue
281
+ try:
282
+ turn_response = self.gemini_session.receive()
283
+ async for chunk in turn_response:
284
+ if not self.is_running:
285
+ break
286
+ if audio_data := chunk.data:
287
+ if not audio_from_gemini_playback_q.full():
288
+ audio_from_gemini_playback_q.put_nowait(audio_data)
289
+ else:
290
+ logging.warning(
291
+ "Audio playback queue full, discarding Gemini audio data.")
292
+ if text_response := chunk.text:
293
+ logging.info(f"Gemini text response: {text_response[:100]}")
294
+ except types.generation_types.StopCandidateException:
295
+ logging.info("Gemini response stream ended normally.")
296
+ except Exception as e:
297
+ if self.is_running:
298
+ logging.error(
299
+ f"Error receiving from Gemini: {e}", exc_info=True)
300
+ await asyncio.sleep(0.1)
301
+ except asyncio.CancelledError:
302
+ logging.info("Task cancelled: process_gemini_responses.")
303
+ finally:
304
+ logging.info("Task finished: process_gemini_responses.")
305
+
306
+ async def play_gemini_audio(self):
307
+ logging.info("Task started: Play Gemini audio responses.")
308
+ if pya is None:
309
+ logging.warning(
310
+ "PyAudio not available. Audio playback task will not run.")
311
+ return
312
+
313
+ try:
314
+ while audio_from_gemini_playback_q is None and self.is_running:
315
+ await asyncio.sleep(0.1)
316
+ if not self.is_running:
317
+ return
318
+
319
+ self.playback_stream = await asyncio.to_thread(
320
+ pya.open, format=PYAUDIO_FORMAT, channels=PYAUDIO_CHANNELS, rate=GEMINI_AUDIO_RECEIVE_SAMPLE_RATE, output=True, frames_per_buffer=PYAUDIO_PLAYBACK_CHUNK_SIZE
321
+ )
322
+ logging.info(
323
+ f"PyAudio playback stream opened at {GEMINI_AUDIO_RECEIVE_SAMPLE_RATE} Hz.")
324
+ while self.is_running:
325
+ try:
326
+ audio_chunk = await asyncio.wait_for(audio_from_gemini_playback_q.get(), timeout=1.0)
327
+ if audio_chunk is None and not self.is_running:
328
+ break # Sentinel and stop signal
329
+ if audio_chunk:
330
+ await asyncio.to_thread(self.playback_stream.write, audio_chunk)
331
+ if audio_chunk:
332
+ audio_from_gemini_playback_q.task_done()
333
+ except asyncio.TimeoutError:
334
+ continue
335
+ except Exception as e:
336
+ logging.error(f"Error playing audio chunk: {e}", exc_info=True)
337
+ await asyncio.sleep(0.01)
338
+ except Exception as e:
339
+ logging.error(
340
+ f"Failed to open or use PyAudio playback stream (might be expected in this environment): {e}", exc_info=True)
341
+ finally:
342
+ if self.playback_stream:
343
+ logging.info("Stopping and closing PyAudio playback stream.")
344
+ try:
345
+ await asyncio.to_thread(self.playback_stream.stop_stream)
346
+ await asyncio.to_thread(self.playback_stream.close)
347
+ except Exception as e_close:
348
+ logging.error(
349
+ f"Error closing playback stream: {e_close}", exc_info=True)
350
+ self.playback_stream = None
351
+ logging.info("Task finished: play_gemini_audio.")
352
+
353
+ def signal_stop(self):
354
+ logging.info("Signal to stop GeminiInteractionLoop received.")
355
+ self.is_running = False
356
+ for q_name, q_obj_ref in [("video_q", video_frames_to_gemini_q),
357
+ ("audio_in_q", audio_chunks_to_gemini_q),
358
+ ("audio_out_q", audio_from_gemini_playback_q)]:
359
+ if q_obj_ref:
360
+ try:
361
+ q_obj_ref.put_nowait(None)
362
+ except asyncio.QueueFull:
363
+ logging.warning(
364
+ f"Queue {q_name} was full when trying to put sentinel for stop signal.")
365
+ except Exception as e:
366
+ logging.error(
367
+ f"Error putting sentinel in {q_name}: {e}", exc_info=True)
368
+
369
+ async def run_main_loop(self):
370
+ global video_frames_to_gemini_q, audio_chunks_to_gemini_q, audio_from_gemini_playback_q
371
+
372
+ self.async_event_loop = asyncio.get_running_loop()
373
+ self.is_running = True
374
+ logging.info("GeminiInteractionLoop run_main_loop starting...")
375
+
376
+ video_frames_to_gemini_q = asyncio.Queue(
377
+ maxsize=MEDIA_TO_GEMINI_QUEUE_MAXSIZE)
378
+ audio_chunks_to_gemini_q = asyncio.Queue(
379
+ maxsize=MEDIA_TO_GEMINI_QUEUE_MAXSIZE)
380
+ audio_from_gemini_playback_q = asyncio.Queue(
381
+ maxsize=AUDIO_PLAYBACK_QUEUE_MAXSIZE)
382
+ logging.info("Asyncio queues initialized in GeminiInteractionLoop.")
383
+
384
+ if client is None:
385
+ logging.critical(
386
+ "Gemini client is None in run_main_loop. Aborting.")
387
+ return
388
+
389
+ try:
390
+ async with client.aio.live.connect(model=MODEL_NAME, config=LIVE_CONNECT_CONFIG) as session:
391
+ self.gemini_session = session
392
+ logging.info(
393
+ f"Gemini session established with API for model {MODEL_NAME}.")
394
+ try:
395
+ logging.info("Sending system prompt to Gemini...")
396
+ await self.gemini_session.send_client_content(content=[types.Part(text=MEDICAL_ASSISTANT_SYSTEM_PROMPT)], end_of_turn=True)
397
+ logging.info("System prompt sent successfully.")
398
+ except Exception as e:
399
+ logging.error(
400
+ f"Failed to send system prompt: {e}", exc_info=True)
401
+ self.is_running = False
402
+ return
403
+
404
+ tasks = []
405
+ try:
406
+ logging.info("Creating async tasks for Gemini interaction...")
407
+ media_stream_task = asyncio.create_task(self.stream_media_to_gemini(), name="stream_media_to_gemini")
408
+ response_process_task = asyncio.create_task(self.process_gemini_responses(), name="process_gemini_responses")
409
+ audio_play_task = asyncio.create_task(self.play_gemini_audio(), name="play_gemini_audio")
410
+ tasks = [media_stream_task, response_process_task, audio_play_task]
411
+ logging.info("All Gemini interaction tasks created.")
412
+
413
+ # Wait for all tasks to complete, collecting all results/exceptions
414
+ results = await asyncio.gather(*tasks, return_exceptions=True)
415
+
416
+ for i, result in enumerate(results):
417
+ if isinstance(result, Exception):
418
+ task_name = tasks[i].get_name() if hasattr(tasks[i], 'get_name') else f"Task-{i}"
419
+ logging.error(f"Task '{task_name}' failed: {result}", exc_info=result)
420
+ # If one task fails, we might want to signal others to stop.
421
+ # self.signal_stop() # This is already called in finally, but could be earlier if needed.
422
+ except asyncio.CancelledError:
423
+ logging.info("One or more tasks were cancelled during gather.")
424
+ except Exception as e_gather:
425
+ logging.error(f"Error during task management with asyncio.gather: {e_gather}", exc_info=True)
426
+ finally:
427
+ # Ensure all tasks are cancelled if not already done, before main loop finally block
428
+ for task in tasks:
429
+ if task and not task.done():
430
+ task.cancel()
431
+ # Await their cancellation (or completion if they finished cleanly before cancel)
432
+ if tasks: # Ensure tasks list is not empty
433
+ await asyncio.gather(*tasks, return_exceptions=True) # Suppress errors from already handled/cancelled tasks
434
+ logging.info("Gemini interaction tasks processing completed or handled.")
435
+
436
+ except websockets.exceptions.ConnectionClosedError as e:
437
+ logging.error(f"WebSocket connection closed with error code {e.code}: {e}")
438
+ st.error(f"Connection to Gemini API failed: {e}. Please try again.")
439
+ except asyncio.CancelledError:
440
+ logging.info("GeminiInteractionLoop.run_main_loop() was cancelled.")
441
+ except Exception as e: # General catch-all
442
+ logging.error(
443
+ f"Exception in GeminiInteractionLoop run_main_loop: {type(e).__name__}: {e}", exc_info=True)
444
+ finally:
445
+ logging.info("GeminiInteractionLoop.run_main_loop() finishing...")
446
+ self.is_running = False
447
+ self.signal_stop() # Ensure sentinels are sent
448
+
449
+ self.gemini_session = None
450
+ video_frames_to_gemini_q = None
451
+ audio_chunks_to_gemini_q = None
452
+ audio_from_gemini_playback_q = None
453
+ logging.info(
454
+ "GeminiInteractionLoop finished and global queues set to None.")
455
+
456
+
457
+ # --- WebRTC Media Processors ---
458
+ class VideoProcessor(VideoProcessorBase):
459
+ def __init__(self):
460
+ self.frame_counter = 0
461
+ self.last_gemini_send_time = time.monotonic()
462
+
463
+ async def _process_and_queue_frame_async(self, frame_ndarray):
464
+ if video_frames_to_gemini_q is None:
465
+ return
466
+ self.frame_counter += 1
467
+ current_time = time.monotonic()
468
+ if (current_time - self.last_gemini_send_time) < (1.0 / VIDEO_FPS_TO_GEMINI):
469
+ return
470
+ self.last_gemini_send_time = current_time
471
+ try:
472
+ img_rgb = cv2.cvtColor(frame_ndarray, cv2.COLOR_BGR2RGB)
473
+ pil_img = PIL.Image.fromarray(img_rgb)
474
+ pil_img.thumbnail(VIDEO_API_RESIZE) # Smaller resolution
475
+ image_io = io.BytesIO()
476
+ pil_img.save(image_io, format="jpeg", quality=85) # Lower quality
477
+ image_bytes = image_io.getvalue()
478
+
479
+ # Check if image size is too large before encoding to base64
480
+ if len(image_bytes) > MAX_PAYLOAD_SIZE_BYTES:
481
+ logging.warning(f"Image too large ({len(image_bytes)} bytes), reducing quality further")
482
+ image_io = io.BytesIO()
483
+ pil_img.save(image_io, format="jpeg", quality=60) # Even lower quality
484
+ image_bytes = image_io.getvalue()
485
+
486
+ api_data = {"mime_type": "image/jpeg",
487
+ "data": base64.b64encode(image_bytes).decode()}
488
+
489
+ if video_frames_to_gemini_q.full():
490
+ try:
491
+ await asyncio.wait_for(video_frames_to_gemini_q.get(), timeout=0.01)
492
+ except asyncio.TimeoutError:
493
+ logging.warning("Video queue full, frame dropped.")
494
+ return
495
+ video_frames_to_gemini_q.put_nowait(api_data)
496
+ except Exception as e:
497
+ logging.error(
498
+ f"Error processing/queueing video frame: {e}", exc_info=True)
499
+
500
+ async def recv(self, frame):
501
+ img_bgr = frame.to_ndarray(format="bgr24")
502
+ try:
503
+ loop = asyncio.get_running_loop()
504
+ loop.create_task(self._process_and_queue_frame_async(img_bgr))
505
+ except RuntimeError:
506
+ logging.error(
507
+ "VideoProcessor.recv: No running asyncio loop in current thread for create_task.")
508
+ return frame
509
+
510
+
511
+ class AudioProcessor(AudioProcessorBase):
512
+ async def _process_and_queue_audio_async(self, audio_frames):
513
+ if audio_chunks_to_gemini_q is None:
514
+ return
515
+ for frame in audio_frames:
516
+ audio_data = frame.planes[0].to_bytes()
517
+
518
+ # Skip empty audio frames
519
+ if not audio_data or len(audio_data) == 0:
520
+ continue
521
+
522
+ # Fix for the WebSocket error 1007 (invalid payload data)
523
+ # Use the correct mime type format and ensure the audio data is valid
524
+ # The audio format must match one of the formats supported by the Gemini API
525
+ # Using standard audio/L16 with 16kHz sample rate instead of 24kHz
526
+ mime_type = f"audio/L16;rate=16000;channels=1"
527
+
528
+ try:
529
+ # Prepare API data - making sure all data is valid
530
+ if isinstance(audio_data, bytes) and len(audio_data) > 0:
531
+ api_data = {"data": audio_data, "mime_type": mime_type}
532
+
533
+ if audio_chunks_to_gemini_q.full():
534
+ try:
535
+ await asyncio.wait_for(audio_chunks_to_gemini_q.get(), timeout=0.01)
536
+ except asyncio.TimeoutError:
537
+ logging.warning("Audio queue full, chunk dropped.")
538
+ continue
539
+ audio_chunks_to_gemini_q.put_nowait(api_data)
540
+ except Exception as e:
541
+ logging.error(
542
+ f"Error queueing audio chunk: {e}", exc_info=True)
543
+
544
+ async def recv(self, frames):
545
+ try:
546
+ loop = asyncio.get_running_loop()
547
+ loop.create_task(self._process_and_queue_audio_async(frames))
548
+ except RuntimeError:
549
+ logging.error(
550
+ "AudioProcessor.recv: No running asyncio loop in current thread for create_task.")
551
+ return frames
552
+
553
+
554
+ # --- Streamlit UI and Application Logic ---
555
+ def initialize_app_session_state():
556
+ defaults = {
557
+ 'gemini_session_active': False,
558
+ 'gemini_loop_instance': None,
559
+ 'webrtc_component_key': f"webrtc_streamer_key_{int(time.time())}",
560
+ }
561
+ for key, value in defaults.items():
562
+ if key not in st.session_state:
563
+ st.session_state[key] = value
564
+
565
+
566
+ def run_streamlit_app():
567
+ st.set_page_config(page_title="Voice AI Medical Assistant", layout="wide")
568
+ initialize_app_session_state()
569
+
570
+ st.title("Voice AI Medical Assistant")
571
+
572
+ # Display prominent error if client is not initialized
573
+ if client is None:
574
+ st.error("⚠️ Gemini API key not found or invalid. Please set a valid GEMINI_API_KEY in your .env file.")
575
+ st.info("You can create a .env file in the project directory with content: GEMINI_API_KEY=your_api_key_here")
576
+
577
+ st.warning("IMPORTANT: This is a VOICE-ONLY interface. Speak to the assistant through your microphone.")
578
+ st.info("Remember: This AI cannot provide medical diagnoses. Always consult a healthcare professional for medical advice.")
579
+
580
+ with st.sidebar:
581
+ st.header("Session Control")
582
+ if not st.session_state.gemini_session_active:
583
+ # Fixed emojis
584
+ if st.button("🚀 Start Voice Assistant", type="primary", use_container_width=True, key="start_session_btn"):
585
+ st.session_state.gemini_session_active = True
586
+
587
+ gemini_loop = GeminiInteractionLoop()
588
+ st.session_state.gemini_loop_instance = gemini_loop
589
+ threading.Thread(target=lambda: asyncio.run(gemini_loop.run_main_loop()), name="GeminiLoopThread", daemon=True).start()
590
+ st.success("Voice Assistant starting... Please allow camera/microphone access in your browser if prompted.")
591
+ st.session_state.webrtc_component_key = f"webrtc_streamer_key_{int(time.time())}"
592
+ st.rerun()
593
+ else:
594
+ # Fixed emojis
595
+ if st.button("🛑 Stop Session", type="secondary", use_container_width=True, key="stop_session_btn"):
596
+ if st.session_state.gemini_loop_instance:
597
+ st.session_state.gemini_loop_instance.signal_stop()
598
+ st.session_state.gemini_loop_instance = None
599
+ st.session_state.gemini_session_active = False
600
+ st.warning("Session stopping...")
601
+ time.sleep(0.5)
602
+ st.rerun()
603
+
604
+ if st.session_state.gemini_session_active:
605
+ st.subheader("Your Live Feed (from your browser)")
606
+
607
+ MEDIA_STREAM_CONSTRAINTS = {
608
+ "video": True,
609
+ "audio": {
610
+ "sampleRate": {"ideal": WEBRTC_REQUESTED_SEND_SAMPLE_RATE},
611
+ "channelCount": {"exact": WEBRTC_REQUESTED_AUDIO_CHANNELS},
612
+ "echoCancellation": True,
613
+ "noiseSuppression": True
614
+ }
615
+ }
616
+
617
+ webrtc_ctx = webrtc_streamer(
618
+ key=st.session_state.webrtc_component_key,
619
+ mode=WebRtcMode.SENDONLY,
620
+ rtc_configuration={
621
+ "iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]
622
+ },
623
+ media_stream_constraints=MEDIA_STREAM_CONSTRAINTS,
624
+ video_processor_factory=VideoProcessor,
625
+ audio_processor_factory=AudioProcessor,
626
+ async_processing=True,
627
+ )
628
+
629
+ if webrtc_ctx.state.playing:
630
+ st.success("🎤 Voice Assistant is now ACTIVE. Speak to interact!")
631
+ st.caption("The assistant is listening through your microphone and watching through your camera.")
632
+ elif st.session_state.gemini_session_active:
633
+ st.caption("Connecting... Ensure camera/microphone permissions are granted in your browser.")
634
+ if hasattr(webrtc_ctx.state, 'error') and webrtc_ctx.state.error:
635
+ st.error(f"WebRTC Connection Error: {webrtc_ctx.state.error}")
636
+ else:
637
+ st.info("Click 'Start Voice Assistant' in the sidebar to begin.")
638
+
639
+ # Visual indicator for voice activity
640
+ if st.session_state.gemini_session_active and webrtc_ctx.state.playing:
641
+ with st.container():
642
+ st.markdown("### How to use the Voice Assistant")
643
+ st.markdown("""
644
+ 1. **Speak naturally** - The assistant is listening through your microphone
645
+ 2. **Show things to the camera** - The assistant can see what you're showing
646
+ 3. **Listen for responses** - The assistant will speak back to you
647
+
648
+ You do not need to type anything. This is a completely voice-controlled interface.
649
+ """)
650
+
651
+
652
+ if __name__ == "__main__":
653
+ if client is None:
654
+ logging.critical("Gemini client could not be initialized. Application cannot start.")
655
+ else:
656
+ run_streamlit_app()