Update app.py

app.py

@@ -17,15 +17,21 @@ import json
 import re
 from gradio_client import Client
 from simple_salesforce import Salesforce, SalesforceLogin
+from llama_index.llms.huggingface import HuggingFaceInferenceAPI
 
 
 # Define Pydantic model for incoming request body
 class MessageRequest(BaseModel):
     message: str
-repo_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+# repo_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+# llm_client = InferenceClient(
+#     model=repo_id,
+#     token=os.getenv("HF_TOKEN"),
+# )
+repo_id = "mistralai/Mistral-7B-Instruct-v0.1"
 llm_client = InferenceClient(
     model=repo_id,
-    token=os.getenv("HF_TOKEN"),
+    token=os.getenv("HF_TOKEN")
 )
 
 os.environ["HF_TOKEN"] = os.getenv("HF_TOKEN")
@@ -70,13 +76,20 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
 
 templates = Jinja2Templates(directory="static")
 # Configure Llama index settings
+# Settings.llm = HuggingFaceInferenceAPI(
+#     model_name="meta-llama/Meta-Llama-3-8B-Instruct",
+#     tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct",
+#     context_window=3000,
+#     token=os.getenv("HF_TOKEN"),
+#     max_new_tokens=512,
+#     generate_kwargs={"temperature": 0.1},
+# )
 Settings.llm = HuggingFaceInferenceAPI(
-    model_name="meta-llama/Meta-Llama-3-8B-Instruct",
-    tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct",
-    context_window=3000,
-    token=os.getenv("HF_TOKEN"),
+    model_name="mistralai/Mistral-7B-Instruct-v0.1",
+    token=os.getenv("HF_TOKEN"),  # Your Hugging Face API token
+    context_window=4096,  # Mistral-7B’s context window
     max_new_tokens=512,
-    generate_kwargs={"temperature": 0.1},
+    generate_kwargs={"temperature": 0.1}
 )
 Settings.embed_model = HuggingFaceEmbedding(
     model_name="BAAI/bge-small-en-v1.5"