adjust all frequency

app.py

@@ -316,167 +316,287 @@ def vevo_timbre(content_wav, reference_wav):
     temp_reference_path = "wav/temp_reference.wav"
     output_path = "wav/output_vevotimbre.wav"
     
-    # 检查并正确处理音频数据
+    # 检查并处理音频数据
     if content_wav is None or reference_wav is None:
         raise ValueError("请上传音频文件")
     
-    # Gradio音频组件返回(sample_rate, data)元组或(data, sample_rate)元组
+    # 处理内容音频格式
     if isinstance(content_wav, tuple) and len(content_wav) == 2:
-        # 确保正确的顺序 (data, sample_rate)
         if isinstance(content_wav[0], np.ndarray):
             content_data, content_sr = content_wav
         else:
             content_sr, content_data = content_wav
-        content_tensor = torch.FloatTensor(content_data)
-        if content_tensor.ndim == 1:
-            content_tensor = content_tensor.unsqueeze(0)  # 添加通道维度
+        
+        # 确保是单声道
+        if len(content_data.shape) > 1 and content_data.shape[1] > 1:
+            content_data = np.mean(content_data, axis=1)
+        
+        # 重采样到24kHz
+        if content_sr != 24000:
+            content_tensor = torch.FloatTensor(content_data).unsqueeze(0)
+            content_tensor = torchaudio.functional.resample(content_tensor, content_sr, 24000)
+            content_sr = 24000
+        else:
+            content_tensor = torch.FloatTensor(content_data).unsqueeze(0)
+        
+        # 归一化音量
+        content_tensor = content_tensor / (torch.max(torch.abs(content_tensor)) + 1e-6) * 0.95
     else:
         raise ValueError("内容音频格式不正确")
-        
+    
+    # 处理参考音频格式
     if isinstance(reference_wav, tuple) and len(reference_wav) == 2:
-        # 确保正确的顺序 (data, sample_rate)
         if isinstance(reference_wav[0], np.ndarray):
             reference_data, reference_sr = reference_wav
         else:
             reference_sr, reference_data = reference_wav
-        reference_tensor = torch.FloatTensor(reference_data)
-        if reference_tensor.ndim == 1:
-            reference_tensor = reference_tensor.unsqueeze(0)  # 添加通道维度
+        
+        # 确保是单声道
+        if len(reference_data.shape) > 1 and reference_data.shape[1] > 1:
+            reference_data = np.mean(reference_data, axis=1)
+        
+        # 重采样到24kHz
+        if reference_sr != 24000:
+            reference_tensor = torch.FloatTensor(reference_data).unsqueeze(0)
+            reference_tensor = torchaudio.functional.resample(reference_tensor, reference_sr, 24000)
+            reference_sr = 24000
+        else:
+            reference_tensor = torch.FloatTensor(reference_data).unsqueeze(0)
+        
+        # 归一化音量
+        reference_tensor = reference_tensor / (torch.max(torch.abs(reference_tensor)) + 1e-6) * 0.95
     else:
         raise ValueError("参考音频格式不正确")
     
+    # 打印debug信息
+    print(f"内容音频形状: {content_tensor.shape}, 采样率: {content_sr}")
+    print(f"参考音频形状: {reference_tensor.shape}, 采样率: {reference_sr}")
+    
     # 保存上传的音频
     torchaudio.save(temp_content_path, content_tensor, content_sr)
     torchaudio.save(temp_reference_path, reference_tensor, reference_sr)
     
-    # 获取管道
-    pipeline = get_pipeline("timbre")
-    
-    # 推理
-    gen_audio = pipeline.inference_fm(
-        src_wav_path=temp_content_path,
-        timbre_ref_wav_path=temp_reference_path,
-        flow_matching_steps=32,
-    )
-    
-    # 保存生成的音频
-    save_audio(gen_audio, output_path=output_path)
-    
-    return output_path
+    try:
+        # 获取管道
+        pipeline = get_pipeline("timbre")
+        
+        # 推理
+        gen_audio = pipeline.inference_fm(
+            src_wav_path=temp_content_path,
+            timbre_ref_wav_path=temp_reference_path,
+            flow_matching_steps=32,
+        )
+        
+        # 检查生成音频是否为数值异常
+        if torch.isnan(gen_audio).any() or torch.isinf(gen_audio).any():
+            print("警告：生成的音频包含NaN或Inf值")
+            gen_audio = torch.nan_to_num(gen_audio, nan=0.0, posinf=0.95, neginf=-0.95)
+        
+        print(f"生成音频形状: {gen_audio.shape}, 最大值: {torch.max(gen_audio)}, 最小值: {torch.min(gen_audio)}")
+        
+        # 保存生成的音频
+        save_audio(gen_audio, output_path=output_path)
+        
+        return output_path
+    except Exception as e:
+        print(f"处理过程中出错: {e}")
+        import traceback
+        traceback.print_exc()
+        raise e
 
 def vevo_voice(content_wav, reference_wav):
     temp_content_path = "wav/temp_content.wav"
     temp_reference_path = "wav/temp_reference.wav"
     output_path = "wav/output_vevovoice.wav"
     
-    # 检查并正确处理音频数据
+    # 检查并处理音频数据
     if content_wav is None or reference_wav is None:
         raise ValueError("请上传音频文件")
     
-    # Gradio音频组件返回(sample_rate, data)元组或(data, sample_rate)元组
+    # 处理内容音频格式
     if isinstance(content_wav, tuple) and len(content_wav) == 2:
-        # 确保正确的顺序 (data, sample_rate)
         if isinstance(content_wav[0], np.ndarray):
             content_data, content_sr = content_wav
         else:
             content_sr, content_data = content_wav
-        content_tensor = torch.FloatTensor(content_data)
-        if content_tensor.ndim == 1:
-            content_tensor = content_tensor.unsqueeze(0)  # 添加通道维度
+        
+        # 确保是单声道
+        if len(content_data.shape) > 1 and content_data.shape[1] > 1:
+            content_data = np.mean(content_data, axis=1)
+        
+        # 重采样到24kHz
+        if content_sr != 24000:
+            content_tensor = torch.FloatTensor(content_data).unsqueeze(0)
+            content_tensor = torchaudio.functional.resample(content_tensor, content_sr, 24000)
+            content_sr = 24000
+        else:
+            content_tensor = torch.FloatTensor(content_data).unsqueeze(0)
+        
+        # 归一化音量
+        content_tensor = content_tensor / (torch.max(torch.abs(content_tensor)) + 1e-6) * 0.95
     else:
         raise ValueError("内容音频格式不正确")
-        
+    
+    # 处理参考音频格式
     if isinstance(reference_wav, tuple) and len(reference_wav) == 2:
-        # 确保正确的顺序 (data, sample_rate)
         if isinstance(reference_wav[0], np.ndarray):
             reference_data, reference_sr = reference_wav
         else:
             reference_sr, reference_data = reference_wav
-        reference_tensor = torch.FloatTensor(reference_data)
-        if reference_tensor.ndim == 1:
-            reference_tensor = reference_tensor.unsqueeze(0)  # 添加通道维度
+        
+        # 确保是单声道
+        if len(reference_data.shape) > 1 and reference_data.shape[1] > 1:
+            reference_data = np.mean(reference_data, axis=1)
+        
+        # 重采样到24kHz
+        if reference_sr != 24000:
+            reference_tensor = torch.FloatTensor(reference_data).unsqueeze(0)
+            reference_tensor = torchaudio.functional.resample(reference_tensor, reference_sr, 24000)
+            reference_sr = 24000
+        else:
+            reference_tensor = torch.FloatTensor(reference_data).unsqueeze(0)
+        
+        # 归一化音量
+        reference_tensor = reference_tensor / (torch.max(torch.abs(reference_tensor)) + 1e-6) * 0.95
     else:
         raise ValueError("参考音频格式不正确")
     
+    # 打印debug信息
+    print(f"内容音频形状: {content_tensor.shape}, 采样率: {content_sr}")
+    print(f"参考音频形状: {reference_tensor.shape}, 采样率: {reference_sr}")
+    
     # 保存上传的音频
     torchaudio.save(temp_content_path, content_tensor, content_sr)
     torchaudio.save(temp_reference_path, reference_tensor, reference_sr)
     
-    # 获取管道
-    pipeline = get_pipeline("voice")
-    
-    # 推理
-    gen_audio = pipeline.inference_ar_and_fm(
-        src_wav_path=temp_content_path,
-        src_text=None,
-        style_ref_wav_path=temp_reference_path,
-        timbre_ref_wav_path=temp_reference_path,
-    )
-    
-    # 保存生成的音频
-    save_audio(gen_audio, output_path=output_path)
-    
-    return output_path
+    try:
+        # 获取管道
+        pipeline = get_pipeline("voice")
+        
+        # 推理
+        gen_audio = pipeline.inference_ar_and_fm(
+            src_wav_path=temp_content_path,
+            src_text=None,
+            style_ref_wav_path=temp_reference_path,
+            timbre_ref_wav_path=temp_reference_path,
+        )
+        
+        # 检查生成音频是否为数值异常
+        if torch.isnan(gen_audio).any() or torch.isinf(gen_audio).any():
+            print("警告：生成的音频包含NaN或Inf值")
+            gen_audio = torch.nan_to_num(gen_audio, nan=0.0, posinf=0.95, neginf=-0.95)
+        
+        print(f"生成音频形状: {gen_audio.shape}, 最大值: {torch.max(gen_audio)}, 最小值: {torch.min(gen_audio)}")
+        
+        # 保存生成的音频
+        save_audio(gen_audio, output_path=output_path)
+        
+        return output_path
+    except Exception as e:
+        print(f"处理过程中出错: {e}")
+        import traceback
+        traceback.print_exc()
+        raise e
 
 def vevo_tts(text, ref_wav, timbre_ref_wav=None, src_language="en", ref_language="en"):
     temp_ref_path = "wav/temp_ref.wav"
     temp_timbre_path = "wav/temp_timbre.wav"
     output_path = "wav/output_vevotts.wav"
     
-    # 检查并正确处理音频数据
+    # 检查并处理音频数据
     if ref_wav is None:
         raise ValueError("请上传参考音频文件")
     
-    # Gradio音频组件返回(sample_rate, data)元组或(data, sample_rate)元组
+    # 处理参考音频格式
     if isinstance(ref_wav, tuple) and len(ref_wav) == 2:
-        # 确保正确的顺序 (data, sample_rate)
         if isinstance(ref_wav[0], np.ndarray):
             ref_data, ref_sr = ref_wav
         else:
             ref_sr, ref_data = ref_wav
-        ref_tensor = torch.FloatTensor(ref_data)
-        if ref_tensor.ndim == 1:
-            ref_tensor = ref_tensor.unsqueeze(0)  # 添加通道维度
+        
+        # 确保是单声道
+        if len(ref_data.shape) > 1 and ref_data.shape[1] > 1:
+            ref_data = np.mean(ref_data, axis=1)
+        
+        # 重采样到24kHz
+        if ref_sr != 24000:
+            ref_tensor = torch.FloatTensor(ref_data).unsqueeze(0)
+            ref_tensor = torchaudio.functional.resample(ref_tensor, ref_sr, 24000)
+            ref_sr = 24000
+        else:
+            ref_tensor = torch.FloatTensor(ref_data).unsqueeze(0)
+        
+        # 归一化音量
+        ref_tensor = ref_tensor / (torch.max(torch.abs(ref_tensor)) + 1e-6) * 0.95
     else:
         raise ValueError("参考音频格式不正确")
     
+    # 打印debug信息
+    print(f"参考音频形状: {ref_tensor.shape}, 采样率: {ref_sr}")
+    
     # 保存上传的音频
     torchaudio.save(temp_ref_path, ref_tensor, ref_sr)
     
     if timbre_ref_wav is not None:
         if isinstance(timbre_ref_wav, tuple) and len(timbre_ref_wav) == 2:
-            # 确保正确的顺序 (data, sample_rate)
             if isinstance(timbre_ref_wav[0], np.ndarray):
                 timbre_data, timbre_sr = timbre_ref_wav
             else:
                 timbre_sr, timbre_data = timbre_ref_wav
-            timbre_tensor = torch.FloatTensor(timbre_data)
-            if timbre_tensor.ndim == 1:
-                timbre_tensor = timbre_tensor.unsqueeze(0)  # 添加通道维度
+            
+            # 确保是单声道
+            if len(timbre_data.shape) > 1 and timbre_data.shape[1] > 1:
+                timbre_data = np.mean(timbre_data, axis=1)
+            
+            # 重采样到24kHz
+            if timbre_sr != 24000:
+                timbre_tensor = torch.FloatTensor(timbre_data).unsqueeze(0)
+                timbre_tensor = torchaudio.functional.resample(timbre_tensor, timbre_sr, 24000)
+                timbre_sr = 24000
+            else:
+                timbre_tensor = torch.FloatTensor(timbre_data).unsqueeze(0)
+            
+            # 归一化音量
+            timbre_tensor = timbre_tensor / (torch.max(torch.abs(timbre_tensor)) + 1e-6) * 0.95
+            
+            print(f"音色参考音频形状: {timbre_tensor.shape}, 采样率: {timbre_sr}")
             torchaudio.save(temp_timbre_path, timbre_tensor, timbre_sr)
         else:
             raise ValueError("音色参考音频格式不正确")
     else:
         temp_timbre_path = temp_ref_path
     
-    # 获取管道
-    pipeline = get_pipeline("tts")
-    
-    # 推理
-    gen_audio = pipeline.inference_ar_and_fm(
-        src_wav_path=None,
-        src_text=text,
-        style_ref_wav_path=temp_ref_path,
-        timbre_ref_wav_path=temp_timbre_path,
-        style_ref_wav_text=None,
-        src_text_language=src_language,
-        style_ref_wav_text_language=ref_language,
-    )
-    
-    # 保存生成的音频
-    save_audio(gen_audio, output_path=output_path)
-    
-    return output_path
+    try:
+        # 获取管道
+        pipeline = get_pipeline("tts")
+        
+        # 推理
+        gen_audio = pipeline.inference_ar_and_fm(
+            src_wav_path=None,
+            src_text=text,
+            style_ref_wav_path=temp_ref_path,
+            timbre_ref_wav_path=temp_timbre_path,
+            style_ref_wav_text=None,
+            src_text_language=src_language,
+            style_ref_wav_text_language=ref_language,
+        )
+        
+        # 检查生成音频是否为数值异常
+        if torch.isnan(gen_audio).any() or torch.isinf(gen_audio).any():
+            print("警告：生成的音频包含NaN或Inf值")
+            gen_audio = torch.nan_to_num(gen_audio, nan=0.0, posinf=0.95, neginf=-0.95)
+        
+        print(f"生成音频形状: {gen_audio.shape}, 最大值: {torch.max(gen_audio)}, 最小值: {torch.min(gen_audio)}")
+        
+        # 保存生成的音频
+        save_audio(gen_audio, output_path=output_path)
+        
+        return output_path
+    except Exception as e:
+        print(f"处理过程中出错: {e}")
+        import traceback
+        traceback.print_exc()
+        raise e
 
 # 创建Gradio界面
 with gr.Blocks(title="VEVO Demo") as demo: