Uploading from VS

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

README.md

@@ -0,0 +1,82 @@
+# ChatDocxAI
+
+ChatDocxAI is an advanced document analysis and question-answering system that leverages the power of Google's Gemini AI to provide intelligent insights from your documents. This versatile tool allows you to upload various document formats and engage in natural conversations about their contents. Whether you're analyzing research papers, business reports, or any other document type, ChatDocxAI helps you extract meaningful information through simple questions.
+
+The system employs sophisticated natural language processing techniques to:
+- Break down complex documents into digestible chunks
+- Maintain context awareness across the entire document
+- Provide accurate, contextually relevant answers
+- Handle multiple document formats seamlessly
+
+Perfect for researchers, business analysts, students, and anyone who needs to quickly extract information from documents without manually reading through them.
+
+## Features
+
+- Support for multiple document formats:
+  - PDF (.pdf)
+  - Text (.txt)
+  - Word Documents (.docx)
+  - Excel Spreadsheets (.xlsx)
+  - PowerPoint Presentations (.pptx)
+  - XML files (.xml)
+  - CSV files (.csv)
+  - JSON files (.json)
+- Interactive web interface using Gradio
+- Intelligent document processing and chunking
+- Advanced context retrieval for accurate answers
+- Train on multiple documents at once
+
+## Installation
+
+1. Clone this repository:
+```bash
+git clone https://github.com/yourusername/ChatDocxAI.git
+cd ChatDocxAI
+```
+
+2. Install the required dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+## Usage
+
+1. Run the application:
+```bash
+python app.py
+```
+
+2. The application will start and provide you with two URLs:
+   - A local URL (http://127.0.0.1:7860) for local access
+   - A public URL (https://xxx.gradio.live) that anyone can access
+   
+   You can share the public URL with others to let them interact with your document Q&A system.
+
+3. Upload your documents using the file upload interface
+
+4. Click "Process Document" to analyze your uploaded files
+
+5. Ask questions about your documents in the question box
+
+## How It Works
+
+1. **Document Processing**: When you upload documents, they are processed and split into manageable chunks
+2. **Vector Storage**: The chunks are converted into vectors and stored for efficient retrieval
+3. **Question Handling**: When you ask a question, the system:
+   - Retrieves relevant context from your documents
+   - Builds an intelligent prompt
+   - Uses Gemini AI to generate accurate answers
+
+## Dependencies
+
+- gradio: For the web interface
+- google-genai: For Gemini AI integration
+- langchain: For document processing and chain operations
+- faiss-cpu: For vector storage and retrieval
+- sentence-transformers: For text embeddings
+- unstructured: For document parsing
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+

app.py

@@ -0,0 +1,54 @@
+import gradio as gr
+from utils import (
+    authenticate,
+    split_documents,
+    build_vectorstore,
+    retrieve_context,
+    retrieve_context_approx,
+    build_prompt,
+    ask_gemini,
+    load_documents_gradio,  # Import the new function
+)
+
+client = authenticate()
+store = {"value": None}
+
+
+def upload_and_process(files):
+    if files is None:
+        return "Please upload a file!"
+
+    raw_docs = load_documents_gradio(files)
+    chunks = split_documents(raw_docs)
+    store["value"] = build_vectorstore(chunks)
+    return "Document processed successfully! You can now ask questions."
+
+
+def handle_question(query):
+    if store["value"] is None:
+        return "Please upload and process a document first."
+
+    if store["value"]["chunks"] <= 50:
+        top_chunks = retrieve_context(query, store["value"])
+    else:
+        top_chunks = retrieve_context_approx(query, store["value"])
+
+    prompt = build_prompt(top_chunks, query)
+    answer = ask_gemini(prompt, client)
+    return f"### My Insights :\n\n{answer.strip()}"
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("## Ask Questions from Your Uploaded Documents")
+    file_input = gr.File(label="Upload Your File", file_types=['.pdf', '.txt', '.docx', '.csv', '.json', '.pptx', '.xml', '.xlsx'], file_count='multiple')
+
+    process_btn = gr.Button("Process Document")
+    status = gr.Textbox(label="Processing Status")
+
+    question = gr.Textbox(label="Ask a Question")
+    answer = gr.Markdown()
+
+    process_btn.click(upload_and_process, inputs=file_input, outputs=status)
+    question.submit(handle_question, inputs=question, outputs=answer)
+
+demo.launch(share=True)  # Or demo.deploy(hf_space="your-username/your-space-name")

requirements.txt

@@ -0,0 +1,13 @@
+gradio
+google-genai
+langchain
+langchain-community
+langchain-google-genai
+google-generativeai
+faiss-cpu
+sentence-transformers
+unstructured[pdf]
+unstructured[docx]
+unstructured[ppt]
+unstructured[excel]
+unstructured[xml]

utils.py

@@ -0,0 +1,141 @@
+import os
+import getpass
+import faiss
+import numpy as np
+import warnings
+import logging
+
+# Suppress warnings
+logging.getLogger("pdfminer").setLevel(logging.ERROR)
+warnings.filterwarnings("ignore")
+
+from google import genai
+from google.genai import types
+from sentence_transformers import SentenceTransformer
+from langchain_community.document_loaders import(
+    UnstructuredPDFLoader,
+    TextLoader,
+    CSVLoader,
+    JSONLoader,
+    UnstructuredPowerPointLoader,
+    UnstructuredExcelLoader,
+    UnstructuredXMLLoader,
+    UnstructuredWordDocumentLoader,
+)
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+def authenticate():
+    """Authenticates with the Google Generative AI API using an API key."""
+    api_key = os.environ.get("GOOGLE_API_KEY")
+    if not api_key:
+        api_key = getpass.getpass("Enter your API Key: ")
+    
+    client = genai.Client(api_key=api_key)
+    return client
+
+
+def load_documents_gradio(uploaded_files):
+  docs = []
+  for file in uploaded_files:
+    file_path = file.name
+    # Detect type and load accordingly
+    if file_path.lower().endswith('.pdf'):
+      docs.extend(UnstructuredPDFLoader(file_path).load())
+    elif file_path.lower().endswith('.txt'):
+      docs.extend(TextLoader(file_path).load())
+    elif file_path.lower().endswith('.csv'):
+      docs.extend(CSVLoader(file_path).load())
+    elif file_path.lower().endswith('.json'):
+      docs.extend(JSONLoader(file_path).load())
+    elif file_path.lower().endswith('.pptx'):
+      docs.extend(UnstructuredPowerPointLoader(file_path).load())
+    elif file_path.lower().endswith('.xlsx'):
+      docs.extend(UnstructuredExcelLoader(file_path).load())
+    elif file_path.lower().endswith('.xml'):
+      docs.extend(UnstructuredXMLLoader(file_path).load())
+    elif file_path.lower().endswith('.docx'):
+      docs.extend(UnstructuredWordDocumentLoader(file_path).load())
+    else:
+      print(f'Unsupported File Type: {file_path}')
+  return docs
+
+
+def split_documents(docs, chunk_size=500, chunk_overlap=100):
+    """Splits documents into smaller chunks using RecursiveCharacterTextSplitter."""
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap
+    )
+    return splitter.split_documents(docs)
+
+
+def build_vectorstore(docs, embedding_model_name="all-MiniLM-L6-v2"):
+    """Builds a FAISS vector store from the document chunks."""
+    texts = [doc.page_content.strip() for doc in docs if doc.page_content.strip()]
+    if not texts:
+        raise ValueError("No valid text found in the documents.")
+
+    print(f"No. of Chunks: {len(texts)}")
+
+    model = SentenceTransformer(embedding_model_name)
+    embeddings = model.encode(texts)
+    print(embeddings.shape)
+
+    index = faiss.IndexFlatL2(embeddings.shape[1])
+    index.add(np.array(embeddings).astype("float32"))
+
+    return {
+        "index": index,
+        "texts": texts,
+        "embedding_model": model,
+        "embeddings": embeddings,
+        "chunks": len(texts),
+    }
+
+
+def retrieve_context(query, store, k=6):
+    """Retrieves the top-k context chunks most similar to the query."""
+    query_vec = store["embedding_model"].encode([query])
+    k = min(k, len(store["texts"]))
+    distances, indices = store["index"].search(query_vec, k)
+    return [store["texts"][i] for i in indices[0]]
+
+
+def retrieve_context_approx(query, store, k=6):
+    """Retrieves context chunks using approximate nearest neighbor search."""
+    ncells = 50
+    D = store["index"].d
+    index = faiss.IndexFlatL2(D)
+    nindex = faiss.IndexIVFFlat(index, D, ncells)
+    nindex.nprobe = 10
+
+    if not nindex.is_trained:
+        nindex.train(np.array(store["embeddings"]).astype("float32"))
+
+    nindex.add(np.array(store["embeddings"]).astype("float32"))
+    query_vec = store["embedding_model"].encode([query])
+    k = min(k, len(store["texts"]))
+    _, indices = nindex.search(np.array(query_vec).astype("float32"), k)
+    return [store["texts"][i] for i in indices[0]]
+
+
+def build_prompt(context_chunks, query):
+    """Builds the prompt for the Gemini API using context and query."""
+    context = "\n".join(context_chunks)
+    return f"""You are a highly knowledgeable and helpful assistant. Use the following context to generate a **detailed and step-by-step** answer to the user's question. Include explanations, examples, and reasoning wherever helpful.
+
+Context:
+{context}
+
+Question: {query}
+Answer:"""
+
+
+def ask_gemini(prompt, client):
+    """Calls the Gemini API with the given prompt and returns the response."""
+    response = client.models.generate_content(
+        model="gemini-2.0-flash",  # Or your preferred model
+        contents=[prompt],
+        config=types.GenerateContentConfig(max_output_tokens=2048, temperature=0.5, seed=42),
+    )
+    return response.text