Spaces:
Running
Running
| import gradio as gr | |
| from openai import OpenAI | |
| import os | |
| # Retrieve the access token from the environment variable | |
| ACCESS_TOKEN = os.getenv("HF_TOKEN") | |
| print("Access token loaded.") | |
| # Initialize the OpenAI client with the Hugging Face Inference API endpoint | |
| client = OpenAI( | |
| base_url="https://api-inference.huggingface.co/v1/", | |
| api_key=ACCESS_TOKEN, | |
| ) | |
| print("OpenAI client initialized.") | |
| def respond( | |
| message, | |
| history: list[tuple[str, str]], | |
| system_message, | |
| max_tokens, | |
| temperature, | |
| top_p, | |
| frequency_penalty, | |
| seed, | |
| model, | |
| custom_model | |
| ): | |
| """ | |
| This function handles the chatbot response. It takes in: | |
| - message: the user's new message | |
| - history: the list of previous messages, each as a tuple (user_msg, assistant_msg) | |
| - system_message: the system prompt | |
| - max_tokens: the maximum number of tokens to generate in the response | |
| - temperature: sampling temperature | |
| - top_p: top-p (nucleus) sampling | |
| - frequency_penalty: penalize repeated tokens in the output | |
| - seed: a fixed seed for reproducibility; -1 will mean 'random' | |
| - model: the selected model from the featured list | |
| - custom_model: a custom model specified by the user | |
| """ | |
| print(f"Received message: {message}") | |
| print(f"History: {history}") | |
| print(f"System message: {system_message}") | |
| print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}") | |
| print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}") | |
| print(f"Model: {model}, Custom Model: {custom_model}") | |
| # Determine the model to use | |
| if custom_model.strip() != "": | |
| selected_model = custom_model.strip() | |
| else: | |
| selected_model = model | |
| print(f"Selected model for inference: {selected_model}") | |
| # Convert seed to None if -1 (meaning random) | |
| if seed == -1: | |
| seed = None | |
| # Construct the messages array required by the API | |
| messages = [{"role": "system", "content": system_message}] | |
| # Add conversation history to the context | |
| for val in history: | |
| user_part = val[0] | |
| assistant_part = val[1] | |
| if user_part: | |
| messages.append({"role": "user", "content": user_part}) | |
| print(f"Added user message to context: {user_part}") | |
| if assistant_part: | |
| messages.append({"role": "assistant", "content": assistant_part}) | |
| print(f"Added assistant message to context: {assistant_part}") | |
| # Append the latest user message | |
| messages.append({"role": "user", "content": message}) | |
| # Start with an empty string to build the response as tokens stream in | |
| response = "" | |
| print(f"Sending request to OpenAI API using model: {selected_model}.") | |
| # Make the streaming request to the HF Inference API via openai-like client | |
| for message_chunk in client.chat.completions.create( | |
| model=selected_model, | |
| max_tokens=max_tokens, | |
| stream=True, # Stream the response | |
| temperature=temperature, | |
| top_p=top_p, | |
| frequency_penalty=frequency_penalty, | |
| seed=seed, | |
| messages=messages, | |
| ): | |
| # Extract the token text from the response chunk | |
| token_text = message_chunk.choices[0].delta.content | |
| if token_text is not None: | |
| print(f"Received token: {token_text}") | |
| response += token_text | |
| yield response | |
| print("Completed response generation.") | |
| # Create a Chatbot component with a specified height | |
| chatbot = gr.Chatbot(height=600) | |
| print("Chatbot interface created.") | |
| # Define featured models | |
| featured_models_list = [ | |
| "meta-llama/Llama-3.3-70B-Instruct", | |
| "mistralai/Mistral-7B-v0.1", | |
| "google/gemma-7b", | |
| ] | |
| # Create the Gradio ChatInterface | |
| with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo: | |
| with gr.Tab("Chat"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| # Chat interface | |
| gr.ChatInterface( | |
| respond, | |
| additional_inputs=[ | |
| gr.Textbox(value="", label="System message"), | |
| gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max new tokens"), | |
| gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), | |
| gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-P"), | |
| gr.Slider(minimum=-2.0, maximum=2.0, value=0.0, step=0.1, label="Frequency Penalty"), | |
| gr.Slider(minimum=-1, maximum=65535, value=-1, step=1, label="Seed (-1 for random)"), | |
| gr.Dropdown(label="Featured Models", choices=featured_models_list, value="meta-llama/Llama-3.3-70B-Instruct", interactive=True), | |
| gr.Textbox(value="", label="Custom Model (Optional)"), | |
| ], | |
| fill_height=True, | |
| chatbot=chatbot, | |
| ) | |
| with gr.Column(): | |
| # Featured models accordion | |
| with gr.Accordion("Featured Models", open=True): | |
| model_search = gr.Textbox(label="Filter Models", placeholder="Search for a featured model...", lines=1) | |
| model_radio = gr.Radio(label="Select a model below", choices=featured_models_list, value="meta-llama/Llama-3.3-70B-Instruct", interactive=True) | |
| def filter_models(search_term): | |
| filtered_models = [m for m in featured_models_list if search_term.lower() in m.lower()] | |
| return gr.update(choices=filtered_models) | |
| model_search.change(filter_models, inputs=model_search, outputs=model_radio) | |
| # Custom model textbox | |
| custom_model_textbox = gr.Textbox(label="Custom Model", placeholder="Enter a custom model path here (optional)", lines=1) | |
| with gr.Tab("Information"): | |
| with gr.Accordion("Featured Models", open=False): | |
| gr.HTML( | |
| """ | |
| <p><a href="https://huggingface.co/models?pipeline_tag=text-generation&sort=trending">See all available models</a></p> | |
| <table style="width:100%; text-align:center; margin:auto;"> | |
| <tr> | |
| <th>Model Name</th> | |
| <th>Notes</th> | |
| </tr> | |
| <tr> | |
| <td>meta-llama/Llama-3.3-70B-Instruct</td> | |
| <td>Powerful large language model.</td> | |
| </tr> | |
| <tr> | |
| <td>mistralai/Mistral-7B-v0.1</td> | |
| <td>A smaller, efficient model.</td> | |
| </tr> | |
| <tr> | |
| <td>google/gemma-7b</td> | |
| <td>Google's language model.</td> | |
| </tr> | |
| </table> | |
| """ | |
| ) | |
| with gr.Accordion("Parameters Overview", open=False): | |
| gr.Markdown( | |
| """ | |
| ## Parameters Overview | |
| ### System Message | |
| The system message is an initial instruction or context that you provide to the chatbot. It sets the stage for the conversation and can be used to guide the chatbot's behavior or persona. | |
| ### Max New Tokens | |
| This parameter limits the length of the chatbot's response. It specifies the maximum number of tokens (words or subwords) that the chatbot can generate in a single response. | |
| ### Temperature | |
| Temperature controls the randomness of the chatbot's responses. A higher temperature (e.g., 1.0) makes the output more random and creative, while a lower temperature (e.g., 0.2) makes the output more focused and deterministic. | |
| ### Top-P | |
| Top-P, also known as nucleus sampling, is another way to control the randomness of the responses. It sets a threshold for the cumulative probability of the most likely tokens. The chatbot will only consider tokens whose cumulative probability is below this threshold. | |
| ### Frequency Penalty | |
| This parameter discourages the chatbot from repeating the same tokens or phrases too often. A higher value (e.g., 1.0) penalizes repetition more strongly, while a lower value (e.g., 0.0) has no penalty. | |
| ### Seed | |
| The seed is a number that initializes the random number generator used by the chatbot. If you set a specific seed, you will get the same response every time you run the chatbot with the same parameters. If you set the seed to -1, a random seed will be used, resulting in different responses each time. | |
| ### Featured Models | |
| You can select a featured model from the dropdown list. These models have been pre-selected for their performance and capabilities. | |
| ### Custom Model | |
| If you have a specific model that you want to use, you can enter its path in the Custom Model textbox. This allows you to use models that are not included in the featured list. | |
| """ | |
| ) | |
| print("Gradio interface initialized.") | |
| if __name__ == "__main__": | |
| print("Launching the demo application.") | |
| demo.launch() |