Spaces:
Sleeping
Sleeping
File size: 2,836 Bytes
b61a604 b2f3130 b61a604 b2f3130 b61a604 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
import os
import spaces
import gradio as gr
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from huggingface_hub import HfApi
import time
HF_TOKEN = os.environ.get("HF_TOKEN")
MODELS = ["Qwen/Qwen2.5-Coder-0.5B", "Qwen/Qwen2.5-Coder-1.5B"]
device = "cuda" if torch.cuda.is_available() else "cpu"
model_id = MODELS[0]
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype="auto", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_id)
def load_model(repo_id: str, progress = gr.Progress(track_tqdm=True)):
global model, tokenizer
api = HfApi(token=HF_TOKEN)
if not api.repo_exists(repo_id=repo_id, token=HF_TOKEN): raise gr.Error(f"Model not found: {repo_id}")
model = AutoModelForCausalLM.from_pretrained(repo_id, torch_dtype="auto", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(repo_id)
gr.Info(f"Model loaded {repo_id}")
return repo_id
@spaces.GPU(duration=30)
def infer(message: str, sysprompt: str, tokens: int=30):
messages = [
{"role": "system", "content": sysprompt},
{"role": "user", "content": message}
]
input_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(text=[input_text], return_tensors="pt").to(model.device)
start = time.time()
generated_ids = model.generate(**inputs, max_new_tokens=tokens)
end = time.time()
elapsed_sec = end - start
generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, generated_ids)]
output_str = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(f"Input: {message}")
print(f"Output: {output_str}")
print(f"Elapsed time: {elapsed_sec} sec.")
output_md = f"### {output_str}"
info_md = f"### Elapsed time: {elapsed_sec} sec."
return output_md, info_md
with gr.Blocks() as demo:
model_name = gr.Dropdown(label="Model", choices=MODELS, value=MODELS[0], allow_custom_value=True)
with gr.Row():
message = gr.Textbox(label="Message", value="", lines=1)
sysprompt = gr.Textbox(label="System prompt", value="You are Qwen, created by Alibaba Cloud. You are a helpful assistant.", lines=4)
tokens = gr.Slider(label="Max tokens", value=30, minimum=1, maximum=2048, step=1)
run_button = gr.Button("Run", variant="primary")
output_md = gr.Markdown("<br><br>")
info_md = gr.Markdown("<br><br>")
gr.on(triggers=[run_button.click, message.submit], fn=infer, inputs=[message, sysprompt, tokens], outputs=[output_md, info_md])
#run_button.click(infer, [message, sysprompt, tokens], [output_md, info_md])
model_name.change(load_model, [model_name], [model_name])
demo.launch()
|