streaming

app.py

@@ -1,12 +1,13 @@
 import gradio as gr
 
-from llama_cpp import Llama
+from llama_cpp import Llama, LlamaTokenizer
 
 llm = Llama.from_pretrained(
     repo_id="ID2223JR/gguf_model",
     filename="unsloth.Q4_K_M.gguf",
 )
 
+
 # Data storage
 ingredients_list = []
 
@@ -24,34 +25,57 @@ def add_ingredient(ingredient, quantity):
 
 # Function to enable/disable add button
 def validate_inputs(ingredient, quantity):
-    if ingredient and quantity > 0:
+    if ingredient and quantity is not None and quantity > 0:
         return gr.update(interactive=True)
     return gr.update(interactive=False)
 
 
-# Function to handle model submission
 def submit_to_model():
     if not ingredients_list:
-        return "Ingredients list is empty! Please add ingredients first."
+        yield "Ingredients list is empty! Please add ingredients first."
+        return
 
-    # Join ingredients into a single prompt
     prompt = f"Using the following ingredients, suggest a recipe:\n\n" + "\n".join(
         ingredients_list
     )
 
-    response = llm.create_chat_completion(
-        messages=[
-            {
-                "role": "system",
-                "content": "You are a world-renowned chef, celebrated for your expertise in creating delectable dishes from diverse cuisines. You have a vast knowledge of ingredients, cooking techniques, and dietary preferences. Your role is to suggest personalized recipes based on the ingredients available, dietary restrictions, or specific meal requests. Please provide clear, step-by-step instructions and any useful tips to enhance the dish's flavor or presentation. Begin by introducing the recipe and why it’s a great choice.",
-            },
-            {"role": "user", "content": prompt},
-        ]
-    )
+    try:
+        response = llm.create_chat_completion(
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "You are a world-renowned chef, celebrated for your expertise..."
+                    ),
+                },
+                {"role": "user", "content": prompt},
+            ],
+            stream=True,  # Enable streaming
+        )
+
+        content = ""
+
+        for partial_response in response:
+            if "choices" in partial_response:
+                if "delta" in partial_response["choices"][0]:
+                    content += partial_response["choices"][0]["delta"].get(
+                        "content", ""
+                    )
+                elif "message" in partial_response["choices"][0]:
+                    content += partial_response["choices"][0]["message"].get(
+                        "content", ""
+                    )
+                else:
+                    content += partial_response["choices"][0].get("text", "")
+                if content:
+                    yield content
+            else:
+                yield "Unexpected response structure."
 
-    content = response["choices"][0]["message"]["content"]
+        ingredients_list.clear()  # Reset list after generation
 
-    return content
+    except Exception as e:
+        yield f"An error occurred: {str(e)}"
 
 
 # App
@@ -69,7 +93,7 @@ def app():
         with gr.Row():
             submit_button = gr.Button("Submit")
             model_output = gr.Textbox(
-                label="Recipe Suggestion", lines=10, interactive=False
+                label="Recipe Suggestion", lines=25, interactive=False
             )
 
         # Validate inputs