app.py

import gradio as gr
from openai import OpenAI
import os

# Retrieve the access token from the environment variable
ACCESS_TOKEN = os.getenv("HF_TOKEN")
print("Access token loaded.")

# Initialize the OpenAI client with the Hugging Face Inference API endpoint
client = OpenAI(
    base_url="https://api-inference.huggingface.co/v1/",
    api_key=ACCESS_TOKEN,
)
print("OpenAI client initialized.")

def respond(
    message,
    history: list[tuple[str, str]],
    system_message,
    max_tokens,
    temperature,
    top_p,
    frequency_penalty,
    seed,
    model,
    custom_model
):
    """
    This function handles the chatbot response. It takes in:
    - message: the user's new message
    - history: the list of previous messages, each as a tuple (user_msg, assistant_msg)
    - system_message: the system prompt
    - max_tokens: the maximum number of tokens to generate in the response
    - temperature: sampling temperature
    - top_p: top-p (nucleus) sampling
    - frequency_penalty: penalize repeated tokens in the output
    - seed: a fixed seed for reproducibility; -1 will mean 'random'
    - model: the selected model from the featured list
    - custom_model: a custom model specified by the user
    """

    print(f"Received message: {message}")
    print(f"History: {history}")
    print(f"System message: {system_message}")
    print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}")
    print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}")
    print(f"Model: {model}, Custom Model: {custom_model}")

    # Determine the model to use
    if custom_model.strip() != "":
        selected_model = custom_model.strip()
    else:
        selected_model = model

    print(f"Selected model for inference: {selected_model}")

    # Convert seed to None if -1 (meaning random)
    if seed == -1:
        seed = None

    # Construct the messages array required by the API
    messages = [{"role": "system", "content": system_message}]

    # Add conversation history to the context
    for val in history:
        user_part = val[0]
        assistant_part = val[1]
        if user_part:
            messages.append({"role": "user", "content": user_part})
            print(f"Added user message to context: {user_part}")
        if assistant_part:
            messages.append({"role": "assistant", "content": assistant_part})
            print(f"Added assistant message to context: {assistant_part}")

    # Append the latest user message
    messages.append({"role": "user", "content": message})

    # Start with an empty string to build the response as tokens stream in
    response = ""
    print(f"Sending request to OpenAI API using model: {selected_model}.")

    # Make the streaming request to the HF Inference API via openai-like client
    for message_chunk in client.chat.completions.create(
        model=selected_model,
        max_tokens=max_tokens,
        stream=True,  # Stream the response
        temperature=temperature,
        top_p=top_p,
        frequency_penalty=frequency_penalty,
        seed=seed,
        messages=messages,
    ):
        # Extract the token text from the response chunk
        token_text = message_chunk.choices[0].delta.content
        if token_text is not None:
            print(f"Received token: {token_text}")
            response += token_text
            yield response

    print("Completed response generation.")

# Create a Chatbot component with a specified height
chatbot = gr.Chatbot(height=600)
print("Chatbot interface created.")

# Define featured models
featured_models_list = [
    "meta-llama/Llama-3.3-70B-Instruct",
    "mistralai/Mistral-7B-v0.1",
    "google/gemma-7b",
]

# Create the Gradio ChatInterface
with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo:
    with gr.Tab("Chat"):
        with gr.Row():
            with gr.Column():
                # Chat interface
                gr.ChatInterface(
                    respond,
                    additional_inputs=[
                        gr.Textbox(value="", label="System message"),
                        gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max new tokens"),
                        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
                        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-P"),
                        gr.Slider(minimum=-2.0, maximum=2.0, value=0.0, step=0.1, label="Frequency Penalty"),
                        gr.Slider(minimum=-1, maximum=65535, value=-1, step=1, label="Seed (-1 for random)"),
                        gr.Dropdown(label="Featured Models", choices=featured_models_list, value="meta-llama/Llama-3.3-70B-Instruct", interactive=True),
                        gr.Textbox(value="", label="Custom Model (Optional)"),
                    ],
                    fill_height=True,
                    chatbot=chatbot,
                )
            with gr.Column():
                # Featured models accordion
                with gr.Accordion("Featured Models", open=True):
                    model_search = gr.Textbox(label="Filter Models", placeholder="Search for a featured model...", lines=1)
                    model_radio = gr.Radio(label="Select a model below", choices=featured_models_list, value="meta-llama/Llama-3.3-70B-Instruct", interactive=True)

                    def filter_models(search_term):
                        filtered_models = [m for m in featured_models_list if search_term.lower() in m.lower()]
                        return gr.update(choices=filtered_models)

                    model_search.change(filter_models, inputs=model_search, outputs=model_radio)

                # Custom model textbox
                custom_model_textbox = gr.Textbox(label="Custom Model", placeholder="Enter a custom model path here (optional)", lines=1)

    with gr.Tab("Information"):
        with gr.Accordion("Featured Models", open=False):
            gr.HTML(
                """
                <p><a href="https://huggingface.co/models?pipeline_tag=text-generation&sort=trending">See all available models</a></p>
                <table style="width:100%; text-align:center; margin:auto;">
                    <tr>
                        <th>Model Name</th>
                        <th>Notes</th>
                    </tr>
                    <tr>
                        <td>meta-llama/Llama-3.3-70B-Instruct</td>
                        <td>Powerful large language model.</td>
                    </tr>
                    <tr>
                        <td>mistralai/Mistral-7B-v0.1</td>
                        <td>A smaller, efficient model.</td>
                    </tr>
                    <tr>
                        <td>google/gemma-7b</td>
                        <td>Google's language model.</td>
                    </tr>
                </table>
                """
            )

        with gr.Accordion("Parameters Overview", open=False):
            gr.Markdown(
                """
                ## Parameters Overview

                ### System Message
                The system message is an initial instruction or context that you provide to the chatbot. It sets the stage for the conversation and can be used to guide the chatbot's behavior or persona.

                ### Max New Tokens
                This parameter limits the length of the chatbot's response. It specifies the maximum number of tokens (words or subwords) that the chatbot can generate in a single response.

                ### Temperature
                Temperature controls the randomness of the chatbot's responses. A higher temperature (e.g., 1.0) makes the output more random and creative, while a lower temperature (e.g., 0.2) makes the output more focused and deterministic.

                ### Top-P
                Top-P, also known as nucleus sampling, is another way to control the randomness of the responses. It sets a threshold for the cumulative probability of the most likely tokens. The chatbot will only consider tokens whose cumulative probability is below this threshold.

                ### Frequency Penalty
                This parameter discourages the chatbot from repeating the same tokens or phrases too often. A higher value (e.g., 1.0) penalizes repetition more strongly, while a lower value (e.g., 0.0) has no penalty.

                ### Seed
                The seed is a number that initializes the random number generator used by the chatbot. If you set a specific seed, you will get the same response every time you run the chatbot with the same parameters. If you set the seed to -1, a random seed will be used, resulting in different responses each time.

                ### Featured Models
                You can select a featured model from the dropdown list. These models have been pre-selected for their performance and capabilities.

                ### Custom Model
                If you have a specific model that you want to use, you can enter its path in the Custom Model textbox. This allows you to use models that are not included in the featured list.
                """
            )

print("Gradio interface initialized.")

if __name__ == "__main__":
    print("Launching the demo application.")
    demo.launch()