new dataset

app.py

@@ -6,6 +6,8 @@ from dotenv import load_dotenv
 import datasets
 from langchain.docstore.document import Document
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from langchain_community.document_loaders import HuggingFaceDatasetLoader
 import chromadb
 from chromadb.utils import embedding_functions
 
@@ -90,14 +92,15 @@ class RetrieverTool(Tool):
 
 
 if __name__ == "__main__":
-    knowledge_base = datasets.load_dataset("MuskumPillerum/General-Knowledge")
-
-    source_docs = [
-        Document(
-            page_content=doc["Answer"], metadata={"question": doc["Question"]}
-        )
-        for doc in knowledge_base
-    ]
+    # knowledge_base = datasets.load_dataset("MuskumPillerum/General-Knowledge", split="train")
+    # print(knowledge_base.column_names)
+    # source_docs = [
+    #     Document(
+    #         page_content=doc["Answer"], metadata={"question": doc["Question"]}
+    #     )
+    #     for doc in knowledge_base
+    # ]
+    source_docs = HuggingFaceDatasetLoader("MuskumPillerum/General-Knowledge", "Answer").load()
     text_splitter = RecursiveCharacterTextSplitter(
         chunk_size=500,
         chunk_overlap=50,
@@ -106,40 +109,44 @@ if __name__ == "__main__":
         separators=["\n\n", "\n", ".", " ", ""],
     )
     docs_processed = text_splitter.split_documents(source_docs)
-    # retriever_tool = RetrieverTool(docs_processed)
-    # model = HfApiModel(
-    #     # model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
-    #     model_id="meta-llama/Meta-Llama-3.1-8B-Instruct",
-    #     token=os.getenv("agentic_rag_inference_token"),
-    # )
-    #
-    # agent = CodeAgent(
-    #     tools=[retriever_tool],
-    #     model=model,
-    #     max_steps=10,
-    #     verbosity_level=10,
-    # )
-    # def enter_message(new_message, conversation_history):
-    #     conversation_history.append(gr.ChatMessage(role="user", content=new_message))
-    #     yield "", conversation_history
-    #     for msg in stream_to_gradio(agent, new_message):
-    #         conversation_history.append(msg)
-    #         yield "", conversation_history
-    #
-    #
-    # def clear_message(chat_history: list):
-    #     return chat_history.clear()
-    #
-    # with gr.Blocks() as b:
-    #     chatbot = gr.Chatbot(type="messages", height=1000)
-    #     textbox = gr.Textbox(lines=3, label="")
-    #     enter_button = gr.Button("enter")
-    #     stop_generating_button = gr.Button("stop generating")
-    #     clear_messages_button = gr.Button("clear messages")
-    #     reply_button_click_event = enter_button.click(enter_message, [textbox, chatbot], [textbox, chatbot])
-    #     textbox.submit(enter_message, [textbox, chatbot], [textbox, chatbot])
-    #     clear_messages_button.click(fn=clear_message, inputs=chatbot, outputs=[chatbot, textbox],
-    #                                 cancels=[reply_button_click_event])
-    #     stop_generating_button.click(cancels=[reply_button_click_event])
-    #
-    # b.launch()
+    print(docs_processed)
+    retriever_tool = RetrieverTool(docs_processed)
+    model = HfApiModel(
+        # model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
+        model_id="meta-llama/Meta-Llama-3.1-8B-Instruct",
+        token=os.getenv("agentic_rag_inference_token"),
+    )
+
+    agent = CodeAgent(
+        tools=[retriever_tool],
+        model=model,
+        max_steps=10,
+        verbosity_level=10,
+    )
+
+
+    def enter_message(new_message, conversation_history):
+        conversation_history.append(gr.ChatMessage(role="user", content=new_message))
+        yield "", conversation_history
+        for msg in stream_to_gradio(agent, new_message):
+            conversation_history.append(msg)
+            yield "", conversation_history
+
+
+    def clear_message(chat_history: list):
+        return chat_history.clear()
+
+
+    with gr.Blocks() as b:
+        chatbot = gr.Chatbot(type="messages", height=1000)
+        textbox = gr.Textbox(lines=3, label="")
+        enter_button = gr.Button("enter")
+        stop_generating_button = gr.Button("stop generating")
+        clear_messages_button = gr.Button("clear messages")
+        reply_button_click_event = enter_button.click(enter_message, [textbox, chatbot], [textbox, chatbot])
+        textbox.submit(enter_message, [textbox, chatbot], [textbox, chatbot])
+        clear_messages_button.click(fn=clear_message, inputs=chatbot, outputs=[chatbot, textbox],
+                                    cancels=[reply_button_click_event])
+        stop_generating_button.click(cancels=[reply_button_click_event])
+
+    b.launch()