Spaces:
Paused
Paused
File size: 1,593 Bytes
41d98b6 b5d9a88 eefe540 b5d9a88 41d98b6 eefe540 a0b7dce 41d98b6 eefe540 04fc58a 41d98b6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import gradio as gr
import os
from loguru import logger
from langchain_community.llms import LlamaCpp
from langchain_core.callbacks import CallbackManager, StreamingStdOutCallbackHandler
from langchain_core.prompts import PromptTemplate
import spaces
import json
# Create a directory for logs if it doesn't exist
if not os.path.exists('logs'):
os.makedirs('logs')
# Define the log file path
log_file = 'logs/file_{time}.log'
# Configure the logger to write to the log file
logger.add(log_file, rotation="500 MB")
template = """Question: {question}
Answer: Let's work this out in a step by step way to be sure we have the right answer."""
prompt = PromptTemplate.from_template(template)
# Callbacks support token-wise streaming
callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
# n_gpu_layers = -1 # The number of layers to put on the GPU. The rest will be on the CPU. If you don't know how many layers there are, you can use -1 to move all to GPU.
# n_batch = 512 # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.
# Make sure the model path is correct for your system!
llm = LlamaCpp(
model_path="/home/user/app/models/Phi-3-mini-4k-instruct-q4.gguf",
callback_manager=callback_manager,
verbose=True, # Verbose is required to pass to the callback manager
)
llm_chain = prompt | llm
@spaces.GPU()
def greet(name):
question = name
response = llm_chain.invoke({"question": question})
logger.info(f"Response --> {response}")
return
demo = gr.Interface(fn=greet, inputs="text", outputs="text")
demo.launch()
|