Update app.py

app.py

@@ -8,20 +8,55 @@ import os, json
 from sys import argv
 from vllm import LLM, SamplingParams
 
+# def load_model_processor(model_path):
+#     processor = AutoProcessor.from_pretrained(model_path)
+#     llm = LLM(
+#         model=model_path, trust_remote_code=True, gpu_memory_utilization=0.8,  
+#         enforce_eager=True,  device = "cuda",
+#         limit_mm_per_prompt={"audio": 5},
+#     )
+#     return llm, processor
+
 def load_model_processor(model_path):
     processor = AutoProcessor.from_pretrained(model_path)
-    llm = LLM(
-        model=model_path, trust_remote_code=True, gpu_memory_utilization=0.8,  
-        enforce_eager=True,  device = "cuda",
-        limit_mm_per_prompt={"audio": 5},
-    )
-    return llm, processor
+    model = Qwen2AudioForConditionalGeneration.from_pretrained(model_path, device_map="auto")
+    model_name = model_path.split("/")[-1]
+    return model, processor, model_name
 
 model_path1 = "Qwen/Qwen2-Audio-7B-Instruct" #argv[1]
 model1, processor1 = load_model_processor(model_path1)
 
-def response_to_audio_conv(conversation, model=None, processor=None, temperature = 0.1,repetition_penalty=1.1, top_p = 0.9,
-                    max_new_tokens = 2048):
+# def response_to_audio_conv(conversation, model=None, processor=None, temperature = 0.1,repetition_penalty=1.1, top_p = 0.9,
+#                     max_new_tokens = 2048):
+#     text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
+#     audios = []
+#     for message in conversation:
+#         if isinstance(message["content"], list):
+#             for ele in message["content"]:
+#                 if ele["type"] == "audio":
+#                     if ele['audio_url'] != None:
+#                         audios.append(librosa.load(
+#                             ele['audio_url'], 
+#                             sr=processor.feature_extractor.sampling_rate)[0]
+#                         )
+
+#     sampling_params = SamplingParams(
+#         temperature=temperature, max_tokens=max_new_tokens, repetition_penalty=repetition_penalty, top_p=top_p, top_k=20,
+#         stop_token_ids=[],
+#     )
+
+#     input = {
+#             'prompt': text,
+#             'multi_modal_data': {
+#                 'audio': [(audio, 16000) for audio in audios]
+#             }
+#             }
+
+#     output = model.generate([input], sampling_params=sampling_params)[0]
+#     response = output.outputs[0].text
+#     return response
+
+def response_to_audio_conv(conversation, model=None, processor=None, temperature = 0.1,repetition_penalty=1.1, top_p = 0.9,max_new_tokens = 2048):
     text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
     audios = []
     for message in conversation:
@@ -33,21 +68,15 @@ def response_to_audio_conv(conversation, model=None, processor=None, temperature
                             ele['audio_url'], 
                             sr=processor.feature_extractor.sampling_rate)[0]
                         )
-
-    sampling_params = SamplingParams(
-        temperature=temperature, max_tokens=max_new_tokens, repetition_penalty=repetition_penalty, top_p=top_p, top_k=20,
-        stop_token_ids=[],
-    )
-
-    input = {
-            'prompt': text,
-            'multi_modal_data': {
-                'audio': [(audio, 16000) for audio in audios]
-            }
-            }
-
-    output = model.generate([input], sampling_params=sampling_params)[0]
-    response = output.outputs[0].text
+    if audios != []:
+        inputs = processor(text=text, audios=audios, return_tensors="pt", padding=True,sampling_rate=16000)
+    else: 
+        inputs = processor(text=text, return_tensors="pt", padding=True)
+    inputs.input_ids = inputs.input_ids.to("cuda")
+    inputs = {k: v.to("cuda") for k, v in inputs.items() if v is not None}
+    generate_ids = model.generate(**inputs, max_new_tokens=2048, temperature = 0.3, do_sample=True)
+    generate_ids = generate_ids[:, inputs["input_ids"].size(1):]
+    response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
     return response
 
 def print_like_dislike(x: gr.LikeData):