Update app.py

app.py

@@ -9,6 +9,7 @@ from streaming_stt_nemo import Model
 import torch
 import random
 from openai import OpenAI
+from transformers import AutoProcessor, SeamlessM4TModel
 
 default_lang = "en"
 
@@ -84,47 +85,82 @@ def models(text, model="Llama 3B Service", seed=42):
 async def respond(audio, model, seed):
     user = transcribe(audio)
     reply = models(user, model, seed)
-    # Change the voice to a deep male voice
     communicate = edge_tts.Communicate(reply, voice="en-US-ChristopherNeural")
     with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
         tmp_path = tmp_file.name
         await communicate.save(tmp_path)
     yield tmp_path
 
+# Load the Seamless M4T model and processor
+processor = AutoProcessor.from_pretrained("facebook/seamless-m4t-medium")
+seamless_model = SeamlessM4TModel.from_pretrained("facebook/seamless-m4t-medium")
+
+def translate_speech(audio, target_lang):
+    audio_array, sample_rate = audio
+    inputs = processor(audios=audio_array, return_tensors="pt", sampling_rate=sample_rate)
+    
+    generated_speech = seamless_model.generate(**inputs, tgt_lang=target_lang)
+    
+    translated_text = processor.batch_decode(generated_speech, skip_special_tokens=True)[0]
+    
+    return translated_text
+
 DESCRIPTION = """ # <center><b>Hello, I am Optimus Prime your personal AI voice assistant</b></center>"""
 
 with gr.Blocks(css="style.css") as demo:    
     gr.Markdown(DESCRIPTION)
-    with gr.Row():
-        select = gr.Dropdown([
-            'Llama 3B Service',
-            'Mixtral 8x7B',
-            'Llama 3 8B',
-            'Mistral 7B v0.3',
-            'Phi 3 mini',
-        ],
-        value="Llama 3B Service",
-        label="Model"
-        )
-        seed = gr.Slider(
-        label="Seed",
-        minimum=0,
-        maximum=999999,
-        step=1,
-        value=0,
-        visible=False
-        )
-        input = gr.Audio(label="User", sources="microphone", type="filepath", waveform_options=False)
-        output = gr.Audio(label="AI", type="filepath",
-                        interactive=False,
-                        autoplay=True,
-                        elem_classes="audio")
-        gr.Interface(
-            batch=True,
-            max_batch_size=10, 
-            fn=respond, 
-            inputs=[input, select, seed],
-            outputs=[output], live=True)  
+    
+    with gr.Tabs():
+        with gr.TabItem("Voice Assistant"):
+            with gr.Row():
+                select = gr.Dropdown([
+                    'Llama 3B Service',
+                    'Mixtral 8x7B',
+                    'Llama 3 8B',
+                    'Mistral 7B v0.3',
+                    'Phi 3 mini',
+                ],
+                value="Llama 3B Service",
+                label="Model"
+                )
+                seed = gr.Slider(
+                label="Seed",
+                minimum=0,
+                maximum=999999,
+                step=1,
+                value=0,
+                visible=False
+                )
+                input = gr.Audio(label="User", sources="microphone", type="filepath", waveform_options=False)
+                output = gr.Audio(label="AI", type="filepath",
+                                interactive=False,
+                                autoplay=True,
+                                elem_classes="audio")
+            gr.Interface(
+                batch=True,
+                max_batch_size=10, 
+                fn=respond, 
+                inputs=[input, select, seed],
+                outputs=[output], 
+                live=True
+            )
+        
+        with gr.TabItem("Speech Translation"):
+            with gr.Row():
+                input_audio = gr.Audio(label="Input Speech", sources="microphone", type="numpy", streaming=True)
+                target_lang = gr.Dropdown(
+                    choices=["es", "fr", "de", "it", "ja", "ko", "zh"],
+                    value="es",
+                    label="Target Language"
+                )
+                output_text = gr.Textbox(label="Translated Text")
+            
+            gr.Interface(
+                fn=translate_speech,
+                inputs=[input_audio, target_lang],
+                outputs=[output_text],
+                live=True
+            )
 
 if __name__ == "__main__":
     demo.queue(max_size=200).launch()