Add semantics

.gitattributes

@@ -47,3 +47,17 @@ model/tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
 model/trainer_state.json filter=lfs diff=lfs merge=lfs -text
 model/training_args.bin filter=lfs diff=lfs merge=lfs -text
 model/added_tokens.json filter=lfs diff=lfs merge=lfs -text
+model2/merges.txt filter=lfs diff=lfs merge=lfs -text
+model2/rng_state.pth filter=lfs diff=lfs merge=lfs -text
+model2/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+model2/special_tokens_map.json filter=lfs diff=lfs merge=lfs -text
+model2/training_args.bin filter=lfs diff=lfs merge=lfs -text
+model2/tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
+model2/added_tokens.json filter=lfs diff=lfs merge=lfs -text
+model2/optimizer.pt filter=lfs diff=lfs merge=lfs -text
+model2/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+model2/trainer_state.json filter=lfs diff=lfs merge=lfs -text
+model2/vocab.json filter=lfs diff=lfs merge=lfs -text
+model2/README.md filter=lfs diff=lfs merge=lfs -text
+model2/adapter_config.json filter=lfs diff=lfs merge=lfs -text
+model2/adapter_model.safetensors filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,4 +1,4 @@
-import spaces
+# import spaces
 import os
 import re
 import gradio as gr
@@ -9,19 +9,27 @@ from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration, Text
 import torchaudio
 from threading import Thread
 
-# Model path and configuration
-model_path = "./model"
+# Model paths and configuration
+model_path_1 = "./model"
+model_path_2 = "./model2"
 base_model_id = "Qwen/Qwen2-Audio-7B-Instruct"
 
+# Dictionary to store loaded models and processors
+loaded_models = {}
+
 # Load the model and processor
-def load_model():
+def load_model(model_path):
+    # Check if model is already loaded
+    if model_path in loaded_models:
+        return loaded_models[model_path]
+    
     # Load the processor from the base model
     processor = AutoProcessor.from_pretrained(
         base_model_id,
         trust_remote_code=True,
     )
     
-    # Load the base model
+    # Load the model
     model = Qwen2AudioForConditionalGeneration.from_pretrained(
         model_path,
         torch_dtype=torch.bfloat16,
@@ -31,68 +39,14 @@ def load_model():
     
     model.eval()
     
-    return model, processor
-
-# Initialize model and processor
-model, processor = load_model()
-
-# Function to extract components from model output
-def extract_components(text):
-    thinking = ""
-    semantic = ""
-    answer = ""
-    
-    # Extract thinking
-    think_match = re.search(r"<think>(.*?)</think>", text, re.DOTALL)
-    if think_match:
-        thinking = think_match.group(1).strip()
-    
-    # Extract semantic elements
-    semantic_match = re.search(r"<semantic_elements>(.*?)</semantic_elements>", text, re.DOTALL)
-    if semantic_match:
-        semantic = semantic_match.group(1).strip()
+    # Store in cache
+    loaded_models[model_path] = (model, processor)
     
-    # Extract answer
-    answer_match = re.search(r"<answer>(.*?)</answer>", text, re.DOTALL)
-    if answer_match:
-        answer = answer_match.group(1).strip()
-    
-    return thinking, semantic, answer
+    return model, processor
 
-# Function to handle chat messages
-def chat(message, history):
-    chat = []
-    for item in history:
-        chat.append({"role": "user", "content": item[0]})
-        if item[1] is not None:
-            chat.append({"role": "assistant", "content": item[1]})
-    chat.append({"role": "user", "content": message})
-    messages = processor.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
-    # Tokenize the messages string
-    model_inputs = processor([messages], return_tensors="pt").to(model.device)
-    streamer = TextIteratorStreamer(
-        processor.tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        model_inputs,
-        streamer=streamer,
-        max_new_tokens=1024,
-        do_sample=True,
-        top_p=0.95,
-        top_k=1000,
-        temperature=0.75,
-        num_beams=1,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
+# Initialize first model and processor
+model, processor = load_model(model_path_1)
 
-    # Initialize an empty string to store the generated text
-    partial_text = ""
-    for new_text in streamer:
-        # print(new_text)
-        partial_text += new_text
-        # Yield an empty string to cleanup the message textbox and the updated conversation history
-        yield partial_text
-        
 
 def process_output(output):
     if "<think>" in output:
@@ -106,18 +60,25 @@ def process_output(output):
         output = "<answer>\n" + rest
     elif "</think>" in output:
         rest = output.split("</think>")[0]
-        output = rest + "\n</think>\n"
+        output = rest + "\n</think>\n\n"
     elif "</semantic_elements>" in output:
         rest = output.split("</semantic_elements>")[0]
-        output = rest + "\n</semantic_elements>\n"
+        output = rest + "\n</semantic_elements>\n\n"
     elif "</answer>" in output:
         rest = output.split("</answer>")[0]
         output = rest + "\n</answer>\n"
+    output = output.replace("\\n", "\n")
+    output = output.replace("\\", "\n")
+    output = output.replace("\n-", "-")
     return output
 
 # Keep only the process_audio_streaming function that's actually used in the Gradio interface
-@spaces.GPU
-def process_audio_streaming(audio_file):
+# @spaces.GPU
+def process_audio_streaming(audio_file, model_choice):
+    # Load the selected model
+    model_path = model_path_1 if model_choice == "Think" else model_path_2
+    model, processor = load_model(model_path)
+    
     # Load and process the audio with torchaudio
     waveform, sr = torchaudio.load(audio_file)
     
@@ -182,11 +143,14 @@ def process_audio_streaming(audio_file):
 # Create Gradio interface for audio processing
 audio_demo = gr.Interface(
     fn=process_audio_streaming,
-    inputs=gr.Audio(type="filepath", label="Upload Audio"),
-    outputs=gr.Textbox(label="Generated Output", lines=24),
+    inputs=[
+        gr.Audio(type="filepath", label="Upload Audio"),
+        gr.Radio(["Think", "Think + Semantics"], label="Select Model", value="Think + Semantics")
+    ],
+    outputs=gr.Textbox(label="Generated Output", lines=30),
     title="SemThink",
-    description="Upload an audio file and the model will provide detailed analysis and description.",
-    examples=["examples/1.wav"],  # Add example files here if available
+    description="Upload an audio file and the model will provide detailed analysis and description. Choose between different model versions.",
+    examples=[["examples/1.wav", "Think + Semantics"]],  # Updated default model in examples
     cache_examples=False,
     live=True  # Enable live updates
 )

model2/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a91b1820ee38f2fe4be96b8431300dc9296ec83df43d36f32551cb1bd496b6ac
+size 5102

model2/adapter_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17d8a6c2d1bdf0ef57df3f012388ea935e6871857aa58176e74ceb4f3a9b098a
+size 738

model2/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6fc0e27c7bc6237c5aae1ee934949e8815d3dd25db5094a25a46139bef0875e
+size 22056664

model2/added_tokens.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e3a16457638c3955f95f98446d42eab5096a074daba4dec5d569e2177568a2b
+size 77138

model2/merges.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5
+size 1671853

model2/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf13cbc8b97f25e074014f88000bcaa13df6d80563f3292ef20c558639effca7
+size 44254970

model2/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bf528318c661dc0b61146db44dd65fdefa0d749d5c3cf7ad5b70a3eb0223f43
+size 14244

model2/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d40c58b45a1ae8a479cc013e372bb19d9ca5414b63e6c42a5552daa0c9020545
+size 1064

model2/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1ed3a229905e152acdb6943f501075b5957bd5774c5940edb81ec1b55e86389
+size 57715

model2/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fecdb47d281073055efd605d080013e3114ed0f3c5d8af201e245b199864c9c7
+size 12030779

model2/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ed216fb2f9e3f05ef5d667a2f644a2f91034b500e5224c003f1437247ad8e46
+size 638366

model2/trainer_state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ba1de525be38f9b4f973fea466347ad7214cc12c4a07c9ba0573d6d0eb9b1c3
+size 125458

model2/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2c797b9b82e633c023712edc136372641c83ad01beb94ca50f97b2639489c1e
+size 5880

model2/vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910
+size 2776833