reduction

app.py

@@ -47,14 +47,14 @@ def generate_response(user_input, history):
     ])
     messages += f"\n<|user|>:{user_input}\n<|assistant|>:"
     model_inputs = tokenizer([messages], return_tensors="pt").to(device)
-    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    streamer = TextIteratorStreamer(tokenizer, timeout=5.0, skip_prompt=True, skip_special_tokens=True)
     generate_kwargs = dict(
         **model_inputs,
         streamer=streamer,
-        max_new_tokens=1024,
+        max_new_tokens=256,
         do_sample=True,
-        top_p=0.95,
-        top_k=50,
+        top_p=0.9,
+        top_k=12,
         temperature=0.7,
         num_beams=1,
         stopping_criteria=StoppingCriteriaList([stop])