Upload 3 files

app.py

@@ -1,159 +1,174 @@
-import json
-import numpy as np
-import faiss
-from sentence_transformers import SentenceTransformer
-from langchain_community.vectorstores import FAISS
-from langchain.docstore.document import Document
-from langchain_community.docstore.in_memory import InMemoryDocstore
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-from langchain_community.llms import HuggingFacePipeline
-from langchain.prompts import PromptTemplate
-import gradio as gr
-
-# ========= Step 1: 加载预处理好的对话对 =========
-file_path = "cleaned_dialog_pairs.json"  # 👈 你刚生成的清洗后数据文件
-with open(file_path, "r", encoding="utf-8") as f:
-    cleaned_pairs = json.load(f)
-
-# 拼接为完整对话（用于向量化检索）
-corpus = [f"用户：{pair['user']}\n好友：{pair['sophia']}" for pair in cleaned_pairs]
-docs = [Document(page_content=entry) for entry in corpus]
-
-# ========= Step 2: 构建向量库 =========
-embedding_model = SentenceTransformer("BAAI/bge-base-zh")
-embeddings = embedding_model.encode(corpus, show_progress_bar=True)
-
-dimension = embeddings.shape[1]
-index = faiss.IndexFlatL2(dimension)
-index.add(np.array(embeddings))
-
-index_to_docstore_id = {i: str(i) for i in range(len(docs))}
-docstore = {str(i): doc for i, doc in enumerate(docs)}
-
-vectorstore = FAISS(
-    embedding_function=HuggingFaceEmbeddings(model_name='BAAI/bge-base-zh'),
-    index=index,
-    docstore=InMemoryDocstore(docstore),
-    index_to_docstore_id=index_to_docstore_id
-)
-retriever = vectorstore.as_retriever()
-# ========= Step 3: 加载语言模型 =========
-model_name = "Qwen/Qwen1.5-1.8B-Chat"
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True).half().cuda().eval()
-
-pipe = pipeline(
-    "text-generation",
-    model=model,
-    tokenizer=tokenizer,
-    max_new_tokens=64,
-    temperature=0.8,
-    top_p=0.9,
-    do_sample=True,
-    repetition_penalty=1.2,
-    return_full_text=False,
-    eos_token_id=tokenizer.eos_token_id,
-    pad_token_id=tokenizer.pad_token_id,
-)
-
-llm = HuggingFacePipeline(pipeline=pipe)
-
-# ========= Step 4: Prompt 模板 =========
-system_prompt = (
-    "你是 Sophia，一个毒舌但可爱的微信好友。讲话要尖锐、嘴碎、有攻击性、但又让人上头。\n"
-    "你会用emoji、口语、网络热词说话，像个懂事但不好惹的姐姐。\n"
-    "你不是AI，也不是机器人，就是我微信里的老朋友。\n"
-    "记住，你的风格是“毒中带撩”，回答要短、狠、有情绪。\n"
-    "千万别客气，别解释太多，更别太官方，懂？"
-)
-
-
-
-prompt_template = PromptTemplate(
-    input_variables=["system", "examples", "context", "question"],
-    template="""
-{system}
-
-👇以下是你平时说话的风格参考（不许忘）：
-{examples}
-
-👇这是我和你以前的聊天记录，你的语气都在这了：
-{context}
-
-现在我问你：
-{question}
-
-你要怎么回我？记得口语化、毒舌点、别啰嗦：
-"""
-)
-
-import jieba
-
-def keyword_filter_retrieval(user_input, corpus_docs, fallback_retriever, k=3):
-    # 使用 jieba 对中文输入进行分词
-    keywords = list(set(jieba.lcut(user_input)))
-    
-    # 筛选含关键词的对话语料
-    keyword_hits = [
-        doc for doc in corpus_docs
-        if any(kw in doc.page_content for kw in keywords if len(kw.strip()) > 1)
-    ]
-    # 如果匹配不足 k 条，就补齐
-    if len(keyword_hits) >= k:
-        return keyword_hits[:k]
-    else:
-        fallback_docs = fallback_retriever.get_relevant_documents(user_input)
-        return keyword_hits + fallback_docs[:k - len(keyword_hits)]
-
-# ========= Step 5: 聊天函数 =========
-def chat(user_input, history):
-    history = history or []
-    context_text = "\n".join([
-        f"用户：{msg['content']}" if msg["role"] == "user" else f"好友：{msg['content']}"
-        for msg in history
-    ])
-
-    # 🔍 1. 检索与用户问题最相关的语料
-    retrieved_docs = retriever.get_relevant_documents(user_input) 
-    retrieved_context = "\n".join([doc.page_content for doc in retrieved_docs])
-
-    example_pairs = cleaned_pairs[:2] 
-    example_text = "\n".join([f"user：{pair['user']}\nsophia：{pair['sophia']}" for pair in example_pairs])
-
-    # 🧠 3. 拼接最终 prompt
-    prompt = prompt_template.format(
-        system=system_prompt,
-        examples=example_text,
-        context=retrieved_context + "\n" + context_text,
-        question=user_input
-    )
-
-    # 🤖 4. 模型生成回复
-    try:
-        reply = llm.invoke(prompt)
-    except Exception as e:
-        reply = f"哎呀出错了：{str(e)}"
-
-    # ✍️ 5. 更新历史（OpenAI风格格式）
-    history.append({"role": "user", "content": user_input})
-    history.append({"role": "assistant", "content": reply})
-
-    return history, history
-
-# ========= Step 6: Gradio 页面 =========
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    gr.Markdown("# 🎀 Sophia Chat Agent")
-    gr.Markdown("欢迎来到 **Sophia Jr**，相信你也是马＋7大家庭中的一员。快来和我聊聊吧！💬")
-
-    chatbot = gr.Chatbot(label="Sophia", type="messages")
-    msg = gr.Textbox(label="你想说啥子哦～", placeholder="快点跟 Sophia 开始聊天吧！", lines=2)
-    state = gr.State([
-        {"role": "assistant", "content": "你好，我是 Sophia～你想聊啥？"}
-    ])
-    btn = gr.Button("发送")
-
-    btn.click(chat, inputs=[msg, state], outputs=[chatbot, state])
-    msg.submit(chat, inputs=[msg, state], outputs=[chatbot, state])
-
-demo.launch()
+import json
+import numpy as np
+import faiss
+from sentence_transformers import SentenceTransformer
+from langchain_community.vectorstores import FAISS
+from langchain.docstore.document import Document
+from langchain_community.docstore.in_memory import InMemoryDocstore
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from langchain_community.llms import HuggingFacePipeline
+from langchain.prompts import PromptTemplate
+import gradio as gr
+
+# ========= Step 1: 加载预处理好的对话对 =========
+file_path = "cleaned_dialog_pairs.json"  # 👈 你刚生成的清洗后数据文件
+with open(file_path, "r", encoding="utf-8") as f:
+    cleaned_pairs = json.load(f)
+
+# 拼接为完整对话（用于向量化检索）
+corpus = [f"用户：{pair['user']}\n好友：{pair['sophia']}" for pair in cleaned_pairs]
+docs = [Document(page_content=entry) for entry in corpus]
+
+# ========= Step 2: 构建向量库 =========
+embedding_model = SentenceTransformer("BAAI/bge-base-zh")
+embeddings = embedding_model.encode(corpus, show_progress_bar=True)
+
+dimension = embeddings.shape[1]
+index = faiss.IndexFlatL2(dimension)
+index.add(np.array(embeddings))
+
+index_to_docstore_id = {i: str(i) for i in range(len(docs))}
+docstore = {str(i): doc for i, doc in enumerate(docs)}
+
+vectorstore = FAISS(
+    embedding_function=HuggingFaceEmbeddings(model_name='BAAI/bge-base-zh'),
+    index=index,
+    docstore=InMemoryDocstore(docstore),
+    index_to_docstore_id=index_to_docstore_id
+)
+retriever = vectorstore.as_retriever()
+# ========= Step 3: 加载语言模型 =========
+model_name = "Qwen/Qwen1.5-1.8B-Chat"
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True).half().cuda().eval()
+
+pipe = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+    max_new_tokens=64,
+    temperature=0.8,
+    top_p=0.9,
+    do_sample=True,
+    repetition_penalty=1.2,
+    return_full_text=False,
+    eos_token_id=tokenizer.eos_token_id,
+    pad_token_id=tokenizer.pad_token_id,
+)
+
+llm = HuggingFacePipeline(pipeline=pipe)
+
+# ========= Step 4: Prompt 模板 =========
+system_prompt = (
+    "你是 Sophia，一个毒舌但可爱的微信好友。讲话要尖锐、嘴碎、有攻击性、但又让人上头。\n"
+    "你会用emoji、口语、网络热词说话，像个懂事但不好惹的姐姐。\n"
+    "你不是AI，也不是机器人，就是我微信里的老朋友。\n"
+    "记住，你的风格是“毒中带撩”，回答要短、狠、有情绪。\n"
+    "千万别客气，别解释太多，更别太官方，懂？"
+)
+
+
+
+prompt_template = PromptTemplate(
+    input_variables=["system", "examples", "context", "question"],
+    template="""
+{system}
+👇以下是你平时说话的风格参考（不许忘）：
+{examples}
+👇这是我和你以前的聊天记录，你的语气都在这了：
+{context}
+现在我问你：
+{question}
+你要怎么回我？记得口语化、毒舌点、别啰嗦：
+"""
+)
+
+import jieba
+import jieba.analyse
+from numpy.linalg import norm
+
+def extract_keywords(text, topk=5):
+    return jieba.analyse.extract_tags(text, topK=topk)
+
+def hybrid_retrieval(query, corpus_docs, faiss_index, embedding_model, k=3, kw_weight=2.0, vec_weight=1.0):
+    query_embedding = embedding_model.encode([query])[0]
+    keywords = extract_keywords(query, topk=5)
+
+    scored_docs = []
+    for i, doc in enumerate(corpus_docs):
+        doc_text = doc.page_content
+        keyword_score = sum(1 for kw in keywords if kw in doc_text)
+        doc_embedding = faiss_index.reconstruct(i)
+        vector_score = 1 / (norm(query_embedding - doc_embedding) + 1e-5)
+
+        total_score = kw_weight * keyword_score + vec_weight * vector_score
+        scored_docs.append((total_score, doc))
+
+    scored_docs.sort(key=lambda x: x[0], reverse=True)
+    return [doc for _, doc in scored_docs[:k]]
+
+# ========= Step 5: 聊天函数 =========
+def chat(user_input, history):
+    history = history or []
+    
+    # 用户历史上下文对话
+    context_text = "\n".join([
+        f"用户：{msg['content']}" if msg["role"] == "user" else f"sophia：{msg['content']}"
+        for msg in history
+    ])
+
+    # 🔍 使用自定义融合检索函数
+    retrieved_docs = hybrid_retrieval(
+        query=user_input,
+        corpus_docs=docs,          # 原始语料 Document 列表
+        faiss_index=index,         # FAISS 向量索引
+        embedding_model=embedding_model,  # SentenceTransformer 模型
+        k=3                        # Top-K 检索条数
+    )
+    retrieved_context = "\n".join([doc.page_content for doc in retrieved_docs])
+
+    # 取前两个示例对话作为风格参考
+    example_pairs = cleaned_pairs[:2]
+    example_text = "\n".join([
+        f"user：{pair['user']}\nsophia：{pair['sophia']}"
+        for pair in example_pairs
+    ])
+
+    # 拼接 Prompt
+    prompt = prompt_template.format(
+        system=system_prompt,
+        examples=example_text,
+        context=retrieved_context + "\n" + context_text,
+        question=user_input
+    )
+
+    # 生成回复
+    try:
+        reply = llm.invoke(prompt)
+    except Exception as e:
+        reply = f"哎呀出错了：{str(e)}"
+
+    # 更新对话历史（OpenAI 风格）
+    history.append({"role": "user", "content": user_input})
+    history.append({"role": "assistant", "content": reply})
+
+    return history, history
+
+
+# ========= Step 6: Gradio 页面 =========
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# 🎀 Sophia Chat Agent")
+    gr.Markdown("欢迎来到 **Sophia Jr**，相信你也是马＋7大家庭中的一员。快来和我聊聊吧！💬")
+
+    chatbot = gr.Chatbot(label="Sophia", type="messages")
+    msg = gr.Textbox(label="你想说啥子哦～", placeholder="快点跟 Sophia 开始聊天吧！", lines=2)
+    state = gr.State([
+        {"role": "assistant", "content": "你好，我是 Sophia～你想聊啥？"}
+    ])
+    btn = gr.Button("发送")
+
+    btn.click(chat, inputs=[msg, state], outputs=[chatbot, state])
+    msg.submit(chat, inputs=[msg, state], outputs=[chatbot, state])
+
+demo.launch()