start of some start of agents work

app.py

@@ -12,7 +12,7 @@ from langchain_community.document_loaders import PyMuPDFLoader
 import uuid
 import chainlit as cl
 import os
-from helper_functions import process_file, add_to_qdrant
+from helper_functions import process_file, load_documents_from_url, add_to_qdrant
 
 chat_model = ChatOpenAI(model="gpt-4o-mini")
 te3_small = OpenAIEmbeddings(model="text-embedding-3-small")
@@ -38,24 +38,86 @@ chat_prompt = ChatPromptTemplate.from_messages([("system", rag_system_prompt_tem
 @cl.on_chat_start
 async def on_chat_start():
     qdrant_client = QdrantClient(url=os.environ["QDRANT_ENDPOINT"], api_key=os.environ["QDRANT_API_KEY"])
+    global qdrant_store
     qdrant_store = Qdrant(
         client=qdrant_client,
         collection_name="kai_test_docs",
         embeddings=te3_small
     )
 
+    res = await ask_action()
+    await handle_response(res)
+
+    # Load the style guide from the local file system
+    style_guide_path = "./public/CoExperiences Writing Style Guide V1 (2024).pdf"
+    loader = PyMuPDFLoader(style_guide_path)
+    style_guide_docs = loader.load()
+    style_guide_text = "\n".join([doc.page_content for doc in style_guide_docs])
+    
+    retriever = qdrant_store.as_retriever()
+    global retrieval_augmented_qa_chain
+    retrieval_augmented_qa_chain = (
+        {
+            "context": itemgetter("question") | retriever, 
+            "question": itemgetter("question"),
+            "writing_style_guide": lambda _: style_guide_text
+        }
+        | RunnablePassthrough.assign(context=itemgetter("context"))
+        | chat_prompt
+        | chat_model
+    )
+
+@cl.author_rename
+def rename(orig_author: str):
+    return "AI Assistant"
+
+@cl.on_message
+async def main(message: cl.Message):
+    if message.content.startswith("http://") or message.content.startswith("https://"):
+        message_type = "url"
+    else:
+        message_type = "question"
+    
+    if message_type == "url":
+        # load the file
+        docs = load_documents_from_url(message.content)
+        splits = text_splitter.split_documents(docs)
+        for i, doc in enumerate(splits):
+            doc.metadata["user_upload_source"] = f"source_{i}"
+        print(f"Processing {len(docs)} text chunks")
+
+        # Add to the qdrant_store
+        qdrant_store.add_documents(
+            documents=splits
+        )
+
+        await cl.Message(f"Processing `{response.url}` done. You can now ask questions!").send()
+    
+    else:
+        response = retrieval_augmented_qa_chain.invoke({"question": message.content})
+        await cl.Message(content=response.content).send()
+
+    res = await ask_action()
+    await handle_response(res)
+
+
+## Chainlit helper functions
+async def ask_action():
     res = await cl.AskActionMessage(
         content="Pick an action!",
         actions=[
             cl.Action(name="Question", value="question", label="Ask a question"),
-            cl.Action(name="File", value="file", label="Upload a file or URL"),
+            cl.Action(name="File", value="file", label="Upload a file"),
+            cl.Action(name="Url", value="url", label="Upload a URL"),
         ],
     ).send()
+    return res
 
+async def handle_response(res):
     if res and res.get("value") == "file":
         files = None
         files = await cl.AskFileMessage(
-            content="Please upload a URL, Text, PDF file to begin!",
+            content="Please upload a Text or PDF file to begin!",
             accept=["text/plain", "application/pdf"],
             max_size_mb=12,
         ).send()
@@ -82,33 +144,8 @@ async def on_chat_start():
         msg.content = f"Processing `{file.name}` done. You can now ask questions!"
         await msg.update()
     
+    if res and res.get("value") == "url":
+        await cl.Message(content="Submit a url link in the message box below.").send()
+   
     if res and res.get("value") == "question":
         await cl.Message(content="Ask away!").send()
-    
-    # Load the style guide from the local file system
-    style_guide_path = "./public/CoExperiences Writing Style Guide V1 (2024).pdf"
-    loader = PyMuPDFLoader(style_guide_path)
-    style_guide_docs = loader.load()
-    style_guide_text = "\n".join([doc.page_content for doc in style_guide_docs])
-    
-    retriever = qdrant_store.as_retriever()
-    global retrieval_augmented_qa_chain
-    retrieval_augmented_qa_chain = (
-        {
-            "context": itemgetter("question") | retriever, 
-            "question": itemgetter("question"),
-            "writing_style_guide": lambda _: style_guide_text
-        }
-        | RunnablePassthrough.assign(context=itemgetter("context"))
-        | chat_prompt
-        | chat_model
-    )
-
-@cl.author_rename
-def rename(orig_author: str):
-    return "AI Assistant"
-
-@cl.on_message
-async def main(message: cl.Message):
-    response = retrieval_augmented_qa_chain.invoke({"question": message.content})
-    await cl.Message(content=response.content).send()

helper_functions.py

@@ -1,28 +1,23 @@
 from typing import List
-from langchain_community.document_loaders import PyMuPDFLoader, TextLoader, WebBaseLoader
 from langchain.agents import AgentExecutor, create_openai_functions_agent
+from langchain_community.document_loaders import PyMuPDFLoader, TextLoader, UnstructuredURLLoader, WebBaseLoader
 from langchain_community.vectorstores import Qdrant
 from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_core.language_models import BaseLanguageModel
 import os
 import functools
+import requests
 
-def process_file(file_or_url):
-    if isinstance(file_or_url, str) and file_or_url.startswith(('http://', 'https://')):
-        # Handle URL
-        loader = WebBaseLoader(file_or_url)
-        docs = loader.load()
-        documents.extend(docs)
+def process_file(file):
     # save the file temporarily
-    temp_file = "./"+file_or_url.path
+    temp_file = "./"+file.path
     with open(temp_file, "wb") as file:
-       file.write(file_or_url.content)
-       file_name = file_or_url.name
+       file.write(file.content)
 
     documents = []
-    if file_or_url.path.endswith(".pdf"):
-        loader = PyMuPDFLoader(temp_file)
+    if file.path.endswith(".pdf"):
+        loader = PyMuPDF(temp_file)
         docs = loader.load()
         documents.extend(docs)
     else:
@@ -31,6 +26,41 @@ def process_file(file_or_url):
         documents.extend(docs)
     return documents
 
+def load_documents_from_url(url):
+    try:
+        # Check if it's a PDF
+        if url.endswith(".pdf"):
+            try:
+                loader = PyMuPDFLoader(url)
+                return loader.load()
+            except Exception as e:
+                print(f"Error loading PDF from {url}: {e}")
+                return None
+        
+        # Fetch the content and check for video pages
+        try:
+            response = requests.head(url, timeout=10)  # Timeout for fetching headers
+            content_type = response.headers.get('Content-Type', '')
+        except Exception as e:
+            print(f"Error fetching headers from {url}: {e}")
+            return None
+        
+        # Ignore video content (flagged for now)
+        if 'video' in content_type:
+            return None
+        if 'youtube' in url:
+            return None
+        
+        # Otherwise, treat it as an HTML page
+        try:
+            loader = UnstructuredURLLoader([url])
+            return loader.load()
+        except Exception as e:
+            print(f"Error loading HTML from {url}: {e}")
+            return None
+    except Exception as e:
+        print(f"General error loading from {url}: {e}")
+        return None
 
 def add_to_qdrant(documents, embeddings, qdrant_client, collection_name):
     Qdrant.from_documents(