read in audio

app.py

@@ -3,6 +3,8 @@ import os
 import re
 import gradio as gr
 import torch
+import librosa
+import numpy as np
 from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
 
 # Model path and configuration
@@ -59,13 +61,25 @@ def extract_components(text):
 
 @spaces.GPU
 def process_audio(audio_file):
-    # Load and process the audio
-    sampling_rate = processor.feature_extractor.sampling_rate
+    # Load and process the audio with librosa
+    y, sr = librosa.load(audio_file, sr=None)  # Load audio file
+    
+    # Resample to 16kHz if needed
+    if sr != 16000:
+        y = librosa.resample(y, orig_sr=sr, target_sr=16000)
+        sr = 16000
+    
+    # Convert to mono if stereo
+    if len(y.shape) > 1 and y.shape[1] > 1:
+        y = librosa.to_mono(y)
+            
+    # Set sampling rate for the processor
+    sampling_rate = 16000
     
     # Create conversation format
     conversation = [
         {"role": "user", "content": [
-            {"type": "audio", "audio": audio_file},
+            {"type": "audio", "audio": y},
             {"type": "text", "text": "Describe the audio in detail."}
         ]}
     ]
@@ -76,7 +90,7 @@ def process_audio(audio_file):
     # Process the inputs
     inputs = processor(
         text=chat_text,
-        audios=[audio_file],
+        audios=[y],
         return_tensors="pt",
         sampling_rate=sampling_rate,
     ).to(model.device)

requirements.txt

@@ -3,4 +3,5 @@ torch
 transformers
 peft
 matplotlib
-soundfile
+soundfile
+librosa