support streaming

Signed-off-by: Zhang Jun <[email protected]>

app.py

@@ -4,20 +4,17 @@ from openai import OpenAI
 
 
 title = "ERNIE 4.5 : BAIDU's LLM"
-description = '''
+description = """
 - Official Website: <https://yiyan.baidu.com/> (UI in Chinese)
 - Twitter post: [We've just unveiled ERNIE 4.5 & X1! 🚀](https://x.com/Baidu_Inc/status/1901089355890036897)
 - API services: [Qianfan Large Model Platform](https://cloud.baidu.com/product-s/qianfan_home) (cloud platform providing LLM services, UI in Chinese)
-'''
+"""
 
 
 qianfan_api_key = os.getenv("QIANFAN_TOKEN")
 qianfan_model = "ernie-4.5-8k-preview"
 
-client = OpenAI(
-        base_url='https://qianfan.baidubce.com/v2',
-        api_key=qianfan_api_key
-        )
+client = OpenAI(base_url="https://qianfan.baidubce.com/v2", api_key=qianfan_api_key)
 
 
 def respond(
@@ -39,13 +36,20 @@ def respond(
     messages.append({"role": "user", "content": message})
 
     response = client.chat.completions.create(
-            model=qianfan_model, 
-            messages=messages,
-            max_completion_tokens = max_tokens,
-            temperature = temperature,
-            top_p = top_p
-            )
-    return response.choices[0].message.content
+        model=qianfan_model,
+        messages=messages,
+        max_completion_tokens=max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        stream=True,
+    )
+
+    output_message = ""
+    for chunk in response:
+        token = chunk.choices[0].delta.content
+        output_message += token
+        yield output_message
+
 
 demo = gr.ChatInterface(
     respond,
@@ -61,8 +65,8 @@ demo = gr.ChatInterface(
             label="Top-p (nucleus sampling)",
         ),
     ],
-    title = title,
-    description = description,
+    title=title,
+    description=description,
 )
 
 if __name__ == "__main__":