Remove one chat_with_model function

app.py

@@ -3,7 +3,44 @@ import torch
 import time
 import gradio as gr
 import spaces
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+
+# @spaces.GPU
+# def chat_with_model(messages):
+#     global current_model, current_tokenizer
+#     if current_model is None or current_tokenizer is None:
+#         yield messages + [{"role": "assistant", "content": "⚠️ No model loaded."}]
+#         return
+
+#     current_model.to("cuda").half()
+
+#     prompt = format_prompt(messages)
+#     inputs = current_tokenizer(prompt, return_tensors="pt").to(current_model.device)
+
+#     streamer = TextIteratorStreamer(current_tokenizer, skip_prompt=True, skip_special_tokens=True)
+#     generation_kwargs = dict(
+#         **inputs,
+#         max_new_tokens=256,
+#         do_sample=True,
+#         streamer=streamer
+#     )
+
+#     # Launch generation in a background thread
+#     thread = threading.Thread(target=current_model.generate, kwargs=generation_kwargs)
+#     thread.start()
+
+#     output_text = ""
+#     messages = messages.copy()
+#     messages.append({"role": "assistant", "content": ""})
+
+#     for new_text in streamer:
+#         output_text += new_text
+#         messages[-1]["content"] = output_text
+#         yield messages
+
+#     current_model.to("cpu")
+#     torch.cuda.empty_cache()
+
 
 # Globals
 current_model = None
@@ -62,7 +99,7 @@ with gr.Blocks() as demo:
             yield messages + [{"role": "assistant", "content": "⚠️ No model loaded."}]
             return
 
-        current_model = current_model.half().to("cuda")
+        current_model = current_model.to("cuda").half()
 
         prompt = format_prompt(messages)
         inputs = current_tokenizer(prompt, return_tensors="pt").to(current_model.device)