Upload 2 files

app.py

@@ -1,211 +1,161 @@
+import spaces
 import json
 import subprocess
 from llama_cpp import Llama
 from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
 from llama_cpp_agent.providers import LlamaCppPythonProvider
-from huggingface_hub import hf_hub_download
+from llama_cpp_agent.chat_history import BasicChatHistory
+from llama_cpp_agent.chat_history.messages import Roles
 import gradio as gr
-import logging
-import spaces
-import time
-import torch
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-repo_id = "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF"
-filename = "Meta-Llama-3-8B-Instruct.Q8_0.gguf"
-
-
-try:
-    start_time = time.time()
-    logger.info("Downloading Model....")
-    hf_hub_download(
-        repo_id = repo_id ,
-        filename = filename,
-        local_dir="./model"
-    )
-    end_time = time.time()
-    logger.info(f"Download complete. Time taken : {start_time - end_time} seconds.")
-
-except Exception as e:
-    logger.error(f"Unable to download Model : {e}")
-    raise
-
-def chunk_text(text, chunk_size=5000):
-    """
-    Splits the input text into chunks of specified size.
+from huggingface_hub import hf_hub_download
 
-    Args:
-        text (str): The input text to be chunked.
-        chunk_size (int): The size of each chunk in tokens.
+hf_hub_download(
+    repo_id="bartowski/gemma-2-9b-it-GGUF",
+    filename="gemma-2-9b-it-Q5_K_M.gguf",
+    local_dir="./models"
+)
 
-    Returns:
-        list: A list of text chunks.
-    """
-    words = text.split()
-    chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
-    return chunks
 
-# def combine_responses(responses):
-#     """
-#     Combines the responses from all chunks into a final output string.
 
-#     Args:
-#         responses (list): A list of responses from each chunk.
+hf_hub_download(
+    repo_id="bartowski/gemma-2-27b-it-GGUF",
+    filename="gemma-2-27b-it-Q5_K_M.gguf",
+    local_dir="./models"
+)
 
-#     Returns:
-#         str: The combined output string.
-#     """
-#     combined_output = " ".join(responses)
-#     return combined_output
 
 llm = None
+llm_model = None
+
+@spaces.GPU(duration=120)
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    model,
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+    top_k,
+    repeat_penalty,
+):
+    chat_template = MessagesFormatterType.GEMMA_2
 
-@spaces.GPU(duration = 110)
-def chat_gguf(message : str, history : list, temperature : float, max_new_tokens : int):
-    """
-    Generate a streaming response using the llama3-8b model with chunking.
-    
-    Args:
-        message (str): The input message.
-        history (list): The conversation history used by ChatInterface. - Not used.
-        temperature (float): The temperature for generating the response.
-        max_new_tokens (int): The maximum number of new tokens to generate.
-        
-    Returns:
-        str: The generated response.
-    """
     global llm
-    if llm == None:
-        try:
-            '''Load the Model'''
-
-            start_time = time.time()
-            logger.info("Loading model...")
-            model = Llama(
-                    model_path = f"./model/{filename}",
-                    flash_attn = True,
-                    n_gpu_layers = -1, #all the layers are offloaded, 
-                    n_batch = 512, #prompt processing batch size (higher the value consumes more hardware)
-                    n_ctx = 8192, #max tokens (input + output)
-                    last_n_tokens = 0, #Context from last N tokens
-                )
-            llm  = model
-            end_time = time.time()
-            logger.info(f"Model loaded. Time taken : {start_time - end_time} seconds.")
-            logger.info(f"Device : {torch.cuda.is_available()}")
-
-        except Exception as e:
-            logger.error(f"Model cannot be loaded : {e}")
-            raise
-
-    start_time = time.time()
-    logger.info("Loading provider...")
-    provider = LlamaCppPythonProvider(model)
-
-    settings = provider.get_provider_default_settings()
+    global llm_model
     
-    #set parameters for the loaded model
-    #settings.max_new_tokens = max_new_tokens
-    settings.temperature = temperature
-    settings.stream = True
-
-    #set chat_template_type
-    chat_template = MessagesFormatterType.LLAMA_3
+    if llm is None or llm_model != model:
+        llm = Llama(
+            model_path=f"models/{model}",
+            flash_attn=True,
+            n_gpu_layers=81,
+            n_batch=1024,
+            n_ctx=8192,
+        )
+        llm_model = model
 
-    #set System_Prompt
-    SYS_PROMPT = '''
-        Extract all relevant keywords and add quantity from the following text and format the result in nested JSON, ignoring personal details and focusing only on the scope of work as shown in the example:
-        Good JSON example: {'lobby': {'frcm': {'replace': {'carpet': 1, 'carpet_pad': 1, 'base': 1, 'window_treatments': 1, 'artwork_and_decorative_accessories': 1, 'portable_lighting': 1, 'upholstered_furniture_and_decorative_pillows': 1, 'millwork': 1} } } }
-        Bad JSON example: {'lobby': { 'frcm': { 'replace': [ 'carpet', 'carpet_pad', 'base', 'window_treatments', 'artwork_and_decorative_accessories', 'portable_lighting', 'upholstered_furniture_and_decorative_pillows', 'millwork'] } } }
-        Make sure to fetch details from the provided text and ignore unnecessary information. The response should be in JSON format only, without any additional comments.
-    '''
-    end_time = time.time()
-    logger.info(f"Provider settings updated. Prompt loaded. Time taken : {start_time - end_time} seconds.")
+    provider = LlamaCppPythonProvider(llm)
 
-    start_time = time.time()
-    logger.info("Loading agent...")
-    # Initialize the agent
     agent = LlamaCppAgent(
         provider,
-        system_prompt=SYS_PROMPT,
+        system_prompt=f"{system_message}",
         predefined_messages_formatter_type=chat_template,
-        debug_output=False
+        debug_output=True
     )
-    end_time = time.time()
-    logger.info(f"Agent loaded. Time taken : {start_time - end_time} seconds.")
-
-    # Chunk the input text into smaller chunks
-    chunks = chunk_text(message)
-
-    responses = []
-    start_time = time.time()
-    logger.info("Generating responses...")
-    for chunk in chunks:
-        response = agent.get_chat_response(
-            chunk,
-            llm_sampling_settings=settings,
-            returns_streaming_generator = True, #generate streamer
-            print_output = False
-        )
-
-        responses.append(response)
-    logger.info(f"Responses generated. Time taken : {time.time() - start_time} seconds.")
-
-    output = ""
-    for response in responses:
-        for text in response:
-            output += text
-
-    yield output
-
-DESCRIPTION = '''
-<div>
-<h1 style="text-align: center;">ContenteaseAI custom trained model</h1>
-</div>
-'''
-
-LICENSE = """
-<p/>
----
-For more information, visit our [website](https://contentease.ai).
-"""
-
-PLACEHOLDER = """
-<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
-   <h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">ContenteaseAI Custom AI trained model</h1>
-   <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Enter the text extracted from the PDF:</p>
-</div>
-"""
+    
+    settings = provider.get_provider_default_settings()
+    settings.temperature = temperature
+    settings.top_k = top_k
+    settings.top_p = top_p
+    settings.max_tokens = max_tokens
+    settings.repeat_penalty = repeat_penalty
+    settings.stream = True
 
-css = """
-h1 {
-  text-align: center;
-  display: block;
-}
+    messages = BasicChatHistory()
+
+    for msn in history:
+        user = {
+            'role': Roles.user,
+            'content': msn[0]
+        }
+        assistant = {
+            'role': Roles.assistant,
+            'content': msn[1]
+        }
+        messages.add_message(user)
+        messages.add_message(assistant)
+    
+    stream = agent.get_chat_response(
+        message,
+        llm_sampling_settings=settings,
+        chat_history=messages,
+        returns_streaming_generator=True,
+        print_output=False
+    )
+    
+    outputs = ""
+    for output in stream:
+        outputs += output
+        yield outputs
+
+description = """<p align="center"><a href="https://huggingface.co/spaces/gokaygokay/Gemma-2-llamacpp" target="_blank">[Reference Space]</a></p>
+<p><center>
+<p align="center">Defaults to 9B it (you can switch to other from additional inputs)</p>
+<p><center>
+<a href="https://huggingface.co/google/gemma-2-27b-it" target="_blank">[27B it Model]</a>
+<a href="https://huggingface.co/google/gemma-2-9b-it" target="_blank">[9B it Model]</a>
+<a href="https://huggingface.co/bartowski/gemma-2-27b-it-GGUF" target="_blank">[27B it Model GGUF]</a>
+<a href="https://huggingface.co/bartowski/gemma-2-9b-it-GGUF" target="_blank">[9B it Model GGUF]</a>
+</center></p>
 """
-# Gradio block
-chatbot = gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Gradio ChatInterface')
 
-with gr.Blocks(fill_height=True, css=css) as demo:
-    gr.Markdown(DESCRIPTION)
-
-    gr.ChatInterface(
-        fn=chat_gguf,
-        chatbot=chatbot,
-        fill_height=True,
-        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
-        additional_inputs=[
-            gr.Slider(minimum=0, maximum=1, step=0.1, value=0.95, label="Temperature", render=False),
-            gr.Slider(minimum=128, maximum=2000, step=1, value=700, label="Max new tokens", render=False),
-        ]
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Dropdown([
+                'gemma-2-9b-it-Q5_K_M.gguf',
+                'gemma-2-27b-it-Q5_K_M.gguf'
+            ],
+            value="gemma-2-9b-it-Q5_K_M.gguf",
+            label="Model"
+        ),
+        gr.Textbox(value="You are a helpful assistant.", label="System message"),
+        gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p",
+        ),
+        gr.Slider(
+            minimum=0,
+            maximum=100,
+            value=40,
+            step=1,
+            label="Top-k",
+        ),
+        gr.Slider(
+            minimum=0.0,
+            maximum=2.0,
+            value=1.1,
+            step=0.1,
+            label="Repetition penalty",
+        ),
+    ],
+    retry_btn="Retry",
+    undo_btn="Undo",
+    clear_btn="Clear",
+    submit_btn="Send",
+    title="Chat with Gemma 2 using llama.cpp", 
+    description=description,
+    chatbot=gr.Chatbot(
+        scale=1, 
+        likeable=False,
+        show_copy_button=True
     )
-
-    gr.Markdown(LICENSE)
+)
 
 if __name__ == "__main__":
-    try:
-        demo.launch(show_error=True, debug = True)
-    except Exception as e:
-        logger.error(f"Error launching Gradio demo: {e}")
+    demo.launch()