app.py

import frontmatter
import gradio as gr
import spaces
import torch

from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline

model_id = "AverageBusinessUser/aidapal"
filename = "aidapal-8k.Q4_K_M.gguf"

print("Downloading model")

tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    gguf_file=filename,
    device_map="auto"
)

# Then create the pipeline with the model and tokenizer
pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer)

@spaces.GPU
def greet(name):
    return pipe(name)

demo = gr.Interface(fn=greet, inputs="text", outputs="text",
                    description=frontmatter.load("README.md").content)
demo.launch()