update

app.py

@@ -25,11 +25,11 @@ if torch.cuda.is_available():
     base_model_id = "llm-jp/llm-jp-3-1.8b-instruct"
     tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=True)
     tokenizer.chat_template = "{{bos_token}}{% for message in messages %}{% if message['role'] == 'user' %}{{ '\\n\\n### 前の投稿:\\n' + message['content'] + '' }}{% elif message['role'] == 'system' %}{{ '以下は、SNS上の投稿です。あなたはSNSの投稿生成botとして、次に続く投稿を考えなさい。説明はせず、投稿の内容のみを鉤括弧をつけずに答えよ。' }}{% elif message['role'] == 'assistant' %}{{ '\\n\\n### 次の投稿:\\n' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '\\n\\n### 次の投稿:\\n' }}{% endif %}{% endfor %}"
-    base_model = AutoModelForCausalLM.from_pretrained(
+    model = AutoModelForCausalLM.from_pretrained(
         base_model_id,
         trust_remote_code=True,
     )
-    model = PeftModel.from_pretrained(base_model, model_id, trust_remote_code=True)
+    model.load_adapter(model_id)
     my_pipeline=pipeline(
         task="text-generation",
         model=model,
@@ -52,12 +52,11 @@ def generate(
         {"role": "user", "content": message},
     ]
 
-    t = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     output = my_pipeline(
-        t,
+        messages,
     )
     print(output)
-    yield output[-1]["generated_text"]
+    yield output[-1]["generated_text"][-1]["content"]
 
 demo = gr.ChatInterface(
     fn=generate,