Update app.py

app.py

@@ -6,8 +6,11 @@ For more information on `huggingface_hub` Inference API support, please check th
 """
 import requests
 
+from openai import OpenAI
 
-client = InferenceClient(os.getenv('MODEL_NAME_OR_PATH'))
+client = OpenAI(api_key='123', base_url=os.getenv('MODEL_NAME_OR_PATH'))
+
+#client = InferenceClient(os.getenv('MODEL_NAME_OR_PATH'))
 
 
 def respond(
@@ -33,14 +36,20 @@ def respond(
 
     response = ""
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
+    res = client.chat.completions.create(
+        model='RefalMachine/ruadapt_qwen2.5_7B_ext_u48_instruct',
+        messages=messages,
         temperature=temperature,
         top_p=top_p,
-        #repetition_penalty=repetition_penalty
-    ):
+        max_tokens=max_tokens,
+        stream=True,
+        extra_body={
+            "repetition_penalty": 1.0,
+            "add_generation_prompt": True,
+        }
+    )
+
+    for message in res:
         token = message.choices[0].delta.content
 
         response += token
@@ -69,5 +78,5 @@ demo = gr.ChatInterface(
 
 
 if __name__ == "__main__":
-    print(requests.get(os.getenv('MODEL_NAME_OR_PATH')[:-3] + '/docs'))
+    #print(requests.get(os.getenv('MODEL_NAME_OR_PATH')[:-3] + '/docs'))
     demo.launch(share=True)