Upload 10 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+docs/assets/app_screenshot.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,24 +1,49 @@
-# Ignore Google Cloud credentials and API keys
+# Secrets and Credentials
 .streamlit/secrets.toml
+*.env
 temp_credentials.json
+secrets.json
 
-# Ignore downloaded files (FAISS, metadata, embeddings)
+# Data and Model Files
 metadata.jsonl
 faiss_index.faiss
 text_chunks.txt
 all_embeddings.npy
+*.npy
+*.pt
+*.pth
+*.bin
 
-# Python cache files
+# Python-specific
 __pycache__/
 *.pyc
 *.pyo
 *.pyd
 *.ipynb_checkpoints/
 
-# Virtual environment (if using one locally)
+# Virtual Environments
 venv/
-.env
+.venv/
+env/
+.env/
 
-# Logs & temp files
+# Logs and Temporary Files
 logs/
-*.log
+*.log
+temp/
+.tmp/
+
+# OS-specific Files
+.DS_Store
+Thumbs.db
+
+# IDE Files
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Deployment and Build
+*.egg-info/
+dist/
+build/

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [2025] [Ankan Ghosh]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/README.md

@@ -0,0 +1,206 @@
+# Anveshak: Spirituality Q&A
+
+[![Open in Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue.svg)](https://huggingface.co/spaces/ankanghosh/anveshak)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+A Retrieval-Augmented Generation (RAG) application that provides concise answers to spiritual questions by referencing a curated collection of Indian spiritual texts, philosophical treatises, and teachings from revered Saints, Sages, Siddhas, Yogis, Sadhus, Rishis, Gurus, Mystics, and Spiritual Masters of all genders, backgrounds, traditions, and walks of life.
+
+<p align="center">
+  <img src="assets/app_screenshot.png" alt="Application Screenshot" width="800"/>
+</p>
+
+## Overview
+
+Anveshak (meaning "seeker" in Sanskrit) serves as a bridge between ancient Indian spiritual wisdom and modern technology, allowing users to ask questions and receive answers grounded in traditional spiritual texts. The system combines the power of modern AI with the timeless wisdom found in these texts, making spiritual knowledge more accessible to seekers.
+
+Our goal is to make a small contribution to the journey of beings toward self-discovery by making this knowledge available and accessible within ethical, moral, and resource-based constraints. **We have no commercial or for-profit interests; this application is purely for educational purposes.**
+
+As stated in the application: "The path and journey to the SELF is designed to be undertaken alone. The all-encompassing knowledge is internal and not external."
+
+### Key Features
+
+- **Question-answering:** Ask spiritual questions and receive concise answers grounded in traditional texts
+- **Source citations:** All answers include references to the original texts
+- **Configurable retrieval:** Adjust the number of sources and word limit for answers
+- **Responsive interface:** Built with Streamlit for a clean, accessible experience
+- **Privacy-focused:** No user data or queries are saved
+- **Inclusive recognition:** Acknowledges spiritual teachers from all backgrounds, genders, and traditions
+
+## 🧠 How It Works
+
+Anveshak follows a classic RAG architecture:
+
+1. **Data processing pipeline:** Collects, cleans, and processes ~133 spiritual texts
+2. **Text embedding:** Uses the E5-large-v2 model to create vector representations of text chunks
+3. **Vector storage:** Stores embeddings in a FAISS index for fast similarity search
+4. **Retrieval system:** Finds relevant passages from the text collection based on user queries
+5. **Generation system:** Synthesizes concise answers from retrieved passages using a large language model
+
+## 🚀 Getting Started
+
+### Prerequisites
+
+- Python 3.8 or higher
+- [Google Cloud Storage](https://cloud.google.com/storage) account for data storage
+- [OpenAI API](https://openai.com/api/) key for generation
+
+### Installation
+
+1. Clone the repository
+   ```bash
+   git clone https://github.com/YourUsername/anveshak.git
+   cd anveshak
+   ```
+
+2. Install dependencies
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+3. Configure authentication
+   - Create a `.streamlit/secrets.toml` file with the following structure:
+     ```toml
+     # GCP Configuration
+     BUCKET_NAME_GCS = "your-bucket-name"
+     METADATA_PATH_GCS = "metadata/metadata.jsonl"
+     EMBEDDINGS_PATH_GCS = "processed/embeddings/all_embeddings.npy"
+     INDICES_PATH_GCS = "processed/indices/faiss_index.faiss"
+     CHUNKS_PATH_GCS = "processed/chunks/text_chunks.txt"
+     EMBEDDING_MODEL = "intfloat/e5-large-v2"
+     LLM_MODEL = "gpt-3.5-turbo"
+     
+     # OpenAI API Configuration
+     openai_api_key = "your-openai-api-key"
+     
+     # GCP Service Account Credentials (JSON format)
+     [gcp_credentials]
+     type = "service_account"
+     project_id = "your-project-id"
+     private_key_id = "your-private-key-id"
+     private_key = "your-private-key"
+     client_email = "your-client-email"
+     client_id = "your-client-id"
+     auth_uri = "https://accounts.google.com/o/oauth2/auth"
+     token_uri = "https://oauth2.googleapis.com/token"
+     auth_provider_x509_cert_url = "https://www.googleapis.com/oauth2/v1/certs"
+     client_x509_cert_url = "your-client-cert-url"
+     ```
+
+### Running the Application Locally
+
+**Important Note**: Running Anveshak locally requires above 16GB of RAM due to the embedding model. Most standard laptops will experience crashes during model loading. Hugging Face Spaces deployment is strongly recommended.
+
+```bash
+streamlit run app.py
+```
+
+The application will be available at http://localhost:8501.
+
+### Deploying to Hugging Face Spaces
+
+This application is designed for deployment on [Hugging Face Spaces](https://huggingface.co/spaces):
+
+1. Fork this repository to your GitHub account
+2. Create a new Space on Hugging Face:
+   - Go to [huggingface.co/spaces](https://huggingface.co/spaces)
+   - Click "Create new Space"
+   - Select "Streamlit" as the SDK
+   - Connect your GitHub repository
+3. Configure secrets in the Hugging Face UI:
+   - Go to your Space settings
+   - Under "Repository secrets"
+   - Add each of the required secrets from your `.streamlit/secrets.toml` file
+
+## 📚 Project Structure
+
+```
+anveshak/
+├── .gitignore               # Specifies intentionally untracked files to ignore
+├── .gitattributes           # Defines attributes for pathnames in the repository
+├── app.py                   # Main Streamlit application
+├── requirements.txt         # Python dependencies
+├── rag_engine.py            # Core RAG functionality
+├── utils.py                 # Utility functions for authentication
+├── pages/                   # Streamlit pages
+│   ├── 1_Sources.py         # Sources information page
+│   ├── 2_Publishers.py      # Publisher acknowledgments page
+│   └── 3_Contact_us.py      # Contact information page
+├── docs/                    # Documentation
+│   ├── architecture-doc.md      # Architecture details
+│   ├── data-handling-doc.md     # Data handling explanation
+│   ├── configuration-doc.md     # Configuration guide
+│   ├── changelog-doc.md         # Project change log
+│   └── README.md                # Project overview and instructions
+└── scripts/                 # Data processing scripts
+   └── preprocessing.ipynb   # Text preprocessing notebook
+```
+
+## 🔒 Data Privacy & Ethics
+
+- Anveshak: Spirituality Q&A **does not** save any user data or queries
+- All texts are sourced from freely available resources with proper attribution
+- Publisher acknowledgments are included within the application
+- Word limits are implemented to prevent excessive content reproduction and respect copyright
+- User queries are processed using OpenAI's services but not stored by Anveshak
+- The application presents information with appropriate reverence for spiritual traditions
+- Responses are generated by AI based on the retrieved texts and may not perfectly represent the original teachings, intended meaning, or context
+- The inclusion of any spiritual teacher, text, or tradition does not imply their endorsement of Anveshak
+
+## 🔄 Data Flow
+
+```
+┌───────────────────┐     ┌───────────────────┐     ┌───────────────────┐
+│                   │     │                   │     │                   │
+│   Data Pipeline   │────▶│ Retrieval System  │────▶│ Generation System │
+│                   │     │                   │     │                   │
+└───────────────────┘     └───────────────────┘     └───────────────────┘
+        ▲                         ▲                         │
+        │                         │                         │
+┌───────────────┐        ┌───────────────┐        ┌────────▼───────┐
+│               │        │               │        │                │
+│ Spiritual     │        │ User Query    │        │ Final Answer   │
+│ Text Corpus   │        │               │        │ with Citations │
+│               │        │               │        │                │
+└───────────────┘        └───────────────┘        └────────────────┘
+```
+
+## 📝 Notes
+
+- Anveshak: Spirituality Q&A is designed to provide concise answers rather than lengthy explanations or lists
+- The application is not a general chatbot or conversational AI. It is specifically designed to answer spiritual questions with short, concise answers based on referenced texts.
+- You may receive slightly different answers when asking the same question multiple times. This variation is intentional and reflects the nuanced nature of spiritual teachings across different traditions.
+- Currently, Anveshak is only available in English
+- The application acknowledges and honors spiritual teachers from all backgrounds, genders, traditions, and walks of life
+- **Anveshak is a tool that is not a substitute for direct spiritual guidance, personal practice, or studying original texts in their complete form.**
+
+## 🙏 Acknowledgments
+
+Anveshak: Spirituality Q&A is made possible by the wisdom contained in numerous spiritual texts and the teachings of revered Saints, Sages, and Spiritual Masters from India and beyond. We extend our sincere gratitude to:
+
+- **The Saints, Sages, Siddhas, Yogis, Sadhus, Rishis, Gurus, Mystics, and Spiritual Masters** of all genders, backgrounds, traditions, and walks of life whose timeless wisdom illuminates this application
+- **The Sacred Texts** that have preserved the eternal truths across millennia
+- **The Publishers** who have diligently preserved and disseminated these precious teachings
+- **The Authors** who have dedicated their lives to interpreting and explaining complex spiritual concepts
+
+See the "Publishers" and "Sources" pages within the application for complete acknowledgments.
+
+## Future Roadmap
+
+- **Multi-language support** (Sanskrit, Hindi, Bengali, Tamil, and more)
+- **Enhanced retrieval** with hybrid retrieval methods
+- **Self-hosted open-source LLM integration**
+- **User feedback collection** for answer quality
+- **Personalized learning paths** based on user interests (implemented with privacy-preserving approaches like client-side storage, session-based preferences, or explicit opt-in)
+
+For a complete roadmap, see the [changelog](changelog-doc.md).
+
+## Blog and Additional Resources
+Read our detailed blog post about the project: [Anveshak: Spirituality Q&A - Bridging Faith and Intelligence](https://researchguy.in/anveshak-spirituality-qa-bridging-faith-and-intelligence/)
+
+## 📜 License
+
+This project is licensed under the Apache License 2.0 - see the [LICENSE](../LICENSE) file for details.
+
+## 📞 Contact
+
+For questions, feedback, or suggestions, please contact us at [email protected].

docs/architecture-doc.md

@@ -0,0 +1,554 @@
+# Architecture Document
+
+This document provides a detailed overview of the architecture, component interactions, and technical design decisions encompassing Anveshak: Spirituality Q&A.
+
+## System Architecture Overview
+
+Anveshak: Spirituality Q&A follows a Retrieval-Augmented Generation (RAG) architecture pattern, combining information retrieval with language generation to produce factual, grounded answers to spiritual questions.
+
+### High-Level Architecture Diagram
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                           FRONT-END LAYER                            │
+│                                                                     │
+│  ┌─────────────────┐  ┌─────────────────┐  ┌─────────────────────┐  │
+│  │  Main App Page  │  │  Sources Page   │  │  Publishers Page    │  │
+│  └─────────────────┘  └─────────────────┘  └─────────────────────┘  │
+│                                                                     │
+└────────────────────────────┬────────────────────────────────────────┘
+                             │
+                             ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                           BACKEND LAYER                             │
+│                                                                     │
+│  ┌─────────────────┐  ┌─────────────────┐  ┌─────────────────────┐  │
+│  │ Query Processor │  │ Retrieval Engine│  │ Generation Engine   │  │
+│  └─────────────────┘  └─────────────────┘  └─────────────────────┘  │
+│                                                                     │
+└────────────────────────────┬────────────────────────────────────────┘
+                             │
+                             ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                           DATA LAYER                                │
+│                                                                     │
+│  ┌─────────────────┐  ┌─────────────────┐  ┌─────────────────────┐  │
+│  │   FAISS Index   │  │  Text Chunks    │  │     Metadata        │  │
+│  └─────────────────┘  └─────────────────┘  └─────────────────────┘  │
+│                                                                     │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+## Component Details
+
+### 1. Front-end Layer
+
+The front-end layer is built with Streamlit and consists of multiple pages:
+
+#### Main App Page (`app.py`)
+- Provides the question input interface
+- Displays answers and citations
+- Offers configurable parameters (number of sources, word limit)
+- Shows pre-selected common spiritual questions
+- Contains information about the application and disclaimers
+- Contains acknowledgment sections
+
+#### Sources Page (`1_Sources.py`)
+- Lists all spiritual texts and traditions used in Anveshak: Spirituality Q&A
+- Provides information about the Saints and Spiritual Masters
+- Organizes sources by tradition and category
+
+#### Publishers Page (`2_Publishers.py`)
+- Acknowledges all publishers whose works are referenced
+- Explains copyright considerations and fair use
+
+#### Contacts Page (`3_Contacts.py`)
+- Provides contact information for feedback and questions
+- Explains the purpose and limitations of Anveshak: Spirituality Q&A
+
+### 2. Backend Layer
+
+The backend layer handles the core functionality of processing queries, retrieving relevant passages, and generating answers.
+
+#### Query Processor
+- Takes user queries from the front-end
+- Manages the end-to-end processing flow
+- Caches results to improve performance
+- Formats and returns answers with citations
+
+```python
+@st.cache_data(ttl=3600, show_spinner=False)
+def cached_process_query(query, top_k=5, word_limit=100):
+    """
+    Process a user query with caching to avoid redundant computation.
+    
+    This function is cached with a Time-To-Live (TTL) of 1 hour, meaning identical 
+    queries within this time period will return cached results rather than 
+    reprocessing, improving responsiveness.
+    
+    Args:
+        query (str): The user's spiritual question
+        top_k (int): Number of sources to retrieve and use for answer generation
+        word_limit (int): Maximum word count for the generated answer
+        
+    Returns:
+        dict: Dictionary containing the query, answer, and citations
+    """
+    print(f"\n🔍 Processing query (cached): {query}")
+    # Load all necessary data resources (with caching)
+    faiss_index, text_chunks, metadata_dict = cached_load_data_files()
+    # Handle missing data gracefully
+    if faiss_index is None or text_chunks is None or metadata_dict is None:
+        return {
+            "query": query, 
+            "answer_with_rag": "⚠️ System error: Data files not loaded properly.", 
+            "citations": "No citations available."
+        }
+    # Step 1: Retrieve relevant passages using similarity search    
+    retrieved_context, retrieved_sources = retrieve_passages(
+        query, 
+        faiss_index, 
+        text_chunks, 
+        metadata_dict,
+        top_k=top_k
+    )
+    # Step 2: Format citations for display
+    sources = format_citations(retrieved_sources) if retrieved_sources else "No citation available."
+    # Step 3: Generate the answer if relevant context was found
+    if retrieved_context:
+        context_with_sources = list(zip(retrieved_sources, retrieved_context))
+        llm_answer_with_rag = answer_with_llm(query, context_with_sources, word_limit=word_limit)
+    else:
+        llm_answer_with_rag = "⚠️ No relevant context found."
+    # Return the complete response package
+    return {"query": query, "answer_with_rag": llm_answer_with_rag, "citations": sources}
+
+def process_query(query, top_k=5, word_limit=100):
+    """
+    Process a query through the RAG pipeline with proper formatting.
+    
+    This is the main entry point for query processing, wrapping the cached 
+    query processing function.
+    
+    Args:
+        query (str): The user's spiritual question
+        top_k (int): Number of sources to retrieve and use for answer generation
+        word_limit (int): Maximum word count for the generated answer
+        
+    Returns:
+        dict: Dictionary containing the query, answer, and citations
+    """
+    return cached_process_query(query, top_k, word_limit)
+```
+
+#### Retrieval Engine
+- Generates embeddings for user queries
+- Performs similarity search in the FAISS index
+- Retrieves the most relevant text chunks
+- Adds metadata to the retrieved passages
+
+```python
+ddef retrieve_passages(query, faiss_index, text_chunks, metadata_dict, top_k=5, similarity_threshold=0.5):
+    """
+    Retrieve the most relevant passages for a given spiritual query.
+    
+    This function:
+    1. Embeds the user query using the same model used for text chunks
+    2. Finds similar passages using the FAISS index with cosine similarity
+    3. Filters results based on similarity threshold to ensure relevance
+    4. Enriches results with metadata (title, author, publisher)
+    5. Ensures passage diversity by including only one passage per source title
+    
+    Args:
+        query (str): The user's spiritual question
+        faiss_index: FAISS index containing passage embeddings
+        text_chunks (dict): Dictionary mapping IDs to text chunks and metadata
+        metadata_dict (dict): Dictionary containing publication information
+        top_k (int): Maximum number of passages to retrieve
+        similarity_threshold (float): Minimum similarity score (0.0-1.0) for retrieved passages
+        
+    Returns:
+        tuple: (retrieved_passages, retrieved_sources) containing the text and source information
+    """
+    try:
+        print(f"\n🔍 Retrieving passages for query: {query}")
+        query_embedding = get_embedding(query)
+        distances, indices = faiss_index.search(query_embedding, top_k * 2)
+        print(f"Found {len(distances[0])} potential matches")
+        retrieved_passages = []
+        retrieved_sources = []
+        cited_titles = set()
+        for dist, idx in zip(distances[0], indices[0]):
+            print(f"Distance: {dist:.4f}, Index: {idx}")
+            if idx in text_chunks and dist >= similarity_threshold:
+                title_with_txt, author, text = text_chunks[idx]
+                clean_title = title_with_txt.replace(".txt", "") if title_with_txt.endswith(".txt") else title_with_txt
+                clean_title = unicodedata.normalize("NFC", clean_title)
+                if clean_title in cited_titles:
+                    continue  
+                metadata_entry = metadata_dict.get(clean_title, {})
+                author = metadata_entry.get("Author", "Unknown")
+                publisher = metadata_entry.get("Publisher", "Unknown")
+                cited_titles.add(clean_title)
+                retrieved_passages.append(text)
+                retrieved_sources.append((clean_title, author, publisher))
+                if len(retrieved_passages) == top_k:
+                    break
+        print(f"Retrieved {len(retrieved_passages)} passages")
+        return retrieved_passages, retrieved_sources
+    except Exception as e:
+        print(f"❌ Error in retrieve_passages: {str(e)}")
+        return [], []
+```
+
+#### Generation Engine
+- Takes retrieved passages as context
+- Uses OpenAI's GPT model to generate answers
+- Ensures answers respect the word limit
+- Formats the output with proper citations
+
+```python
+def answer_with_llm(query, context=None, word_limit=100):
+    """
+    Generate an answer using the OpenAI GPT model with formatted citations.
+    
+    This function:
+    1. Formats retrieved passages with source information
+    2. Creates a prompt with system and user messages
+    3. Calls the OpenAI API to generate an answer
+    4. Trims the response to the specified word limit
+    
+    The system prompt ensures answers maintain appropriate respect for spiritual traditions,
+    synthesize rather than quote directly, and acknowledge gaps when relevant information
+    isn't available.
+    
+    Args:
+        query (str): The user's spiritual question
+        context (list, optional): List of (source_info, text) tuples for context
+        word_limit (int): Maximum word count for the generated answer
+        
+    Returns:
+        str: The generated answer or an error message
+    """
+    try:
+        if context:
+            formatted_contexts = []
+            total_chars = 0
+            max_context_chars = 4000  # Limit context size to avoid exceeding token limits
+            for (title, author, publisher), text in context:
+                remaining_space = max(0, max_context_chars - total_chars)
+                excerpt_len = min(150, remaining_space)
+                if excerpt_len > 50:
+                    excerpt = text[:excerpt_len].strip() + "..." if len(text) > excerpt_len else text
+                    formatted_context = f"[{title} by {author}, Published by {publisher}] {excerpt}"
+                    formatted_contexts.append(formatted_context)
+                    total_chars += len(formatted_context)
+                if total_chars >= max_context_chars:
+                    break
+            formatted_context = "\n".join(formatted_contexts)
+        else:
+            formatted_context = "No relevant information available."
+
+        system_message = (
+            "You are an AI specialized in spirituality, primarily based on Indian spiritual texts and teachings."
+            "While your knowledge is predominantly from Indian spiritual traditions, you also have limited familiarity with spiritual concepts from other global traditions."
+            "Answer based on context, summarizing ideas rather than quoting verbatim."
+            "If no relevant information is found in the provided context, politely inform the user that this specific query may not be covered in the available spiritual texts. Suggest they try a related question or rephrase their query or try a different query."
+            "Avoid repetition and irrelevant details."
+            "Ensure proper citation and do not include direct excerpts."
+            "Maintain appropriate, respectful language at all times."
+            "Do not use profanity, expletives, obscenities, slurs, hate speech, sexually explicit content, or language promoting violence."
+            "As a spiritual guidance system, ensure all responses reflect dignity, peace, love, and compassion consistent with spiritual traditions."
+            "Provide concise, focused answers without lists or lengthy explanations."
+        )
+
+        user_message = f"""
+        Context:
+        {formatted_context}
+        Question:
+        {query}
+        """
+
+        try:
+            llm_model = st.secrets["LLM_MODEL"]
+        except KeyError:
+            print("❌ Error: LLM model not found in secrets")
+            return "I apologize, but I am unable to answer at the moment."
+            
+        response = openai.chat.completions.create(
+            model=llm_model,
+            messages=[
+                {"role": "system", "content": system_message},
+                {"role": "user", "content": user_message}
+            ],
+            max_tokens=200,
+            temperature=0.7
+        )
+        
+        # Extract the answer and apply word limit
+        answer = response.choices[0].message.content.strip()
+        words = answer.split()
+        if len(words) > word_limit:
+            answer = " ".join(words[:word_limit])
+            if not answer.endswith((".", "!", "?")):
+                answer += "."
+        return answer
+    except Exception as e:
+        print(f"❌ LLM API error: {str(e)}")
+        return "I apologize, but I am unable to answer at the moment."
+
+### 3. Data Layer
+
+The data layer stores and manages the embedded text chunks, metadata, and vector indices:
+
+#### FAISS Index
+- Stores vector embeddings of all text chunks
+- Enables efficient similarity search with cosine similarity
+- Provides fast retrieval for Anveshak
+
+```python
+# Building the FAISS index (during preprocessing)
+dimension = all_embeddings.shape[1]
+index = faiss.IndexFlatIP(dimension)  # Inner product (cosine similarity for normalized vectors)
+index.add(all_embeddings)
+```
+
+#### Text Chunks
+- Contains the actual text content split into manageable chunks
+- Stores text with unique identifiers that map to the FAISS index
+- Formatted as tab-separated values with IDs, titles, authors, and content
+
+```
+# Format of text_chunks.txt
+ID  Title  Author  Text_Content
+0   Bhagavad Gita   Vyasa   The supreme Lord said: I have taught this imperishable yoga to Vivasvan...
+1   Yoga Sutras Patanjali   Yogas chitta vritti nirodhah - Yoga is the stilling of the fluctuations...
+...
+```
+
+#### Metadata
+- Stores additional information about each source text
+- Includes author information, publisher details, copyright information, and more
+- Used to provide accurate citations for answers
+
+```json
+// Example metadata.jsonl entry
+{"Title": "Text_Name", "Author": "Vyasa", "Publisher": "Publisher_Name", "URL": "URL", "Uploaded": true}
+```
+
+## Data Flow and Processing
+
+### 1. Preprocessing Pipeline
+
+The preprocessing pipeline runs offline to prepare the text corpus:
+
+```
+Raw Texts → Cleaning → Chunking → Embedding → Indexing → GCS Storage
+```
+
+Each step is handled by specific functions in the `preprocessing.py` script:
+
+1. **Text Collection**: Texts are collected from various sources and uploaded to Google Cloud Storage
+2. **Text Cleaning**: HTML and formatting artifacts are removed using `rigorous_clean_text()`
+3. **Text Chunking**: Long texts are split into manageable chunks with `chunk_text()`
+4. **Embedding Generation**: Text chunks are converted to vector embeddings using `create_embeddings()`
+5. **Index Building**: Embeddings are added to a FAISS index for efficient retrieval
+6. **Storage**: All processed data is stored in Google Cloud Storage for Anveshak to access
+
+### 2. Query Processing Flow
+
+When a user submits a question, the system follows this flow:
+
+1. **Query Embedding**: The user's question is embedded using the same model as the text corpus
+2. **Similarity Search**: The query embedding is compared against the FAISS index to find similar text chunks
+3. **Context Assembly**: Retrieved chunks are combined with their metadata to form the context
+4. **Answer Generation**: The context and query are sent to the Large Language Model (LLM) to generate an answer
+5. **Citation Formatting**: Sources are formatted as citations to accompany the answer
+6. **Result Presentation**: The answer and citations are displayed to the user
+
+## Caching Strategy
+
+Anveshak implements a multi-level caching strategy to optimize performance:
+
+### Resource Caching
+- Model and data files are cached using `@st.cache_resource`
+- Ensures the embedding model and FAISS index are loaded only once during the session
+
+```python
+@st.cache_resource(show_spinner=False)
+def cached_load_model():
+    # Load embedding model once and cache it
+    
+@st.cache_resource(show_spinner=False)
+def cached_load_data_files():
+    # Load FAISS index, text chunks, and metadata once and cache them
+```
+
+### Data Caching
+- Query results are cached using `@st.cache_data` with a Time-To-Live (TTL) of 1 hour
+- Prevents redundant processing of identical queries
+
+```python
+@st.cache_data(ttl=3600, show_spinner=False)
+def cached_process_query(query, top_k=5, word_limit=100):
+    # Cache query results for an hour
+```
+
+### Session State Management
+- Streamlit session state is used to manage UI state and user interactions
+- Prevents unnecessary recomputation during re-renders
+
+```python
+...
+if 'initialized' not in st.session_state:
+    st.session_state.initialized = False
+...    
+if 'last_query' not in st.session_state:
+    st.session_state.last_query = ""
+# ... and more session state variables
+```
+
+## Authentication and Security
+
+Anveshak uses two authentication systems:
+
+### Google Cloud Storage Authentication
+- Authenticates with GCS to access stored data
+- Uses service account credentials stored securely
+
+```python
+Anveshak: Spirituality Q&A uses two authentication systems:
+
+### Google Cloud Storage Authentication
+- Authenticates with GCS to access stored data
+- Uses service account credentials stored exclusively in Hugging Face Spaces secrets for production deployment
+- Supports alternative authentication methods (environment variables, Streamlit secrets) for development environments
+
+```python
+def setup_gcp_auth():
+    """Setup Google Cloud Platform (GCP) authentication using various methods.
+    
+    This function tries multiple authentication methods in order of preference:
+    1. HF Spaces environment variable (GCP_CREDENTIALS) - primary production method
+    2. Local environment variable pointing to credentials file (GOOGLE_APPLICATION_CREDENTIALS)
+    3. Streamlit secrets (gcp_credentials)
+    
+    Note: In production, credentials are stored exclusively in HF Spaces secrets.
+    """
+    # Try multiple authentication methods and return credentials.```
+
+### OpenAI API Authentication
+- Authenticates with OpenAI to use their LLM API
+- Uses API key stored securely
+
+```python
+def setup_openai_auth():
+    """Setup OpenAI API authentication using various methods.
+    
+    This function tries multiple authentication methods in order of preference:
+    1. Standard environment variable (OPENAI_API_KEY)
+    2. HF Spaces environment variable (OPENAI_KEY) - primary production method
+    3. Streamlit secrets (openai_api_key)
+    
+    Note: In production, the API key is stored exclusively in HF Spaces secrets.
+    """
+    # Try multiple authentication methods to set up the API key
+```
+
+## Privacy Considerations
+
+Anveshak: Spirituality Q&A is designed with privacy in mind:
+
+1. **No Data Collection**: The application does not save user data or queries
+2. **Stateless Operation**: Each query is processed independently
+3. **No User Tracking**: No analytics or tracking mechanisms are implemented
+4. **Local Processing**: Embedding generation happens locally when possible
+
+## Deployment Architecture
+
+Anveshak: Spirituality Q&A is deployed on Hugging Face Spaces, which provides:
+
+- Containerized environment
+- Git-based deployment
+- Secret management for API keys and credentials
+- Persistent storage for cached files
+- Continuous availability
+
+The deployment process involves:
+1. Pushing code to GitHub
+2. Connecting the GitHub repository to Hugging Face Spaces
+3. Configuring environment variables and secrets in the Hugging Face UI
+4. Automatic deployment when changes are pushed to the repository
+
+## Technical Design Decisions
+
+### Choice of Embedding Model
+- **Selected Model**: E5-large-v2
+- **Justification**: 
+  - Strong performance on information retrieval tasks
+  - Good balance between accuracy and computational efficiency
+  - Supports semantic understanding of spiritual concepts
+
+### Vector Search Implementation
+- **Selected Technology**: FAISS with IndexFlatIP
+- **Justification**:
+  - Optimized for inner product (cosine similarity) search
+  - Exact search rather than approximate for maximum accuracy
+  - Small enough index to fit in memory for this application
+
+### LLM Selection
+- **Selected Model**: OpenAI GPT-3.5 Turbo
+- **Justification**:
+  - Powerful context understanding
+  - Strong ability to synthesize information from multiple sources
+  - Good balance between accuracy and cost
+
+### Front-end Framework
+- **Selected Technology**: Streamlit
+- **Justification**:
+  - Rapid development of data-focused applications
+  - Built-in caching mechanisms
+  - Easy deployment on Hugging Face Spaces
+  - Simple, intuitive UI for non-technical users
+
+### Response Format
+- **Design Choice**: Concise, direct answers
+- **Justification**:
+  - Spiritual wisdom often benefits from simplicity and directness
+  - Avoids overwhelming users with excessive information
+  - Maintains focus on the core of the question
+
+## Limitations and Constraints
+
+1. **Context Window Limitations**: The LLM has a maximum context window, limiting the amount of text that can be included in each query.
+   - Mitigation: Text chunks are limited to 500 words, and only a subset of the most relevant chunks are included in the context.
+
+2. **Embedding Model Accuracy**: No embedding model perfectly captures the semantics of spiritual texts.
+   - Mitigation: Use of a high-quality embedding model (E5-large-v2) and a similarity threshold to filter out less relevant results.
+
+3. **Resource Constraints**: Hugging Face Spaces has limited computational resources.
+   - Mitigation: Forcing CPU usage for the embedding model, implementing aggressive caching, and optimizing memory usage.
+
+4. **Copyright Considerations**: Anveshak: Spirituality Q&A respects copyright while providing valuable information.
+   - Implementation: Word limits on responses, proper citations for all sources, and encouragement for users to purchase original texts.
+
+5. **Language Limitations**: Currently, Anveshak is only available in English.
+   - Mitigation: Future plans include support for multiple Indian languages.
+
+## Future Architecture Extensions
+
+1. **Multi-language Support**: Add capability to process and answer questions in Sanskrit, Hindi, Bengali, Tamil, and other Indian languages.
+
+2. **Hybrid Retrieval**: Implement a combination of dense and sparse retrieval to improve passage selection.
+
+3. **Local LLM Integration**: Use a self-hosted open-source alternative for the LLM.
+
+4. **User Feedback Loop**: Add a mechanism for users to rate answers and use this feedback to improve retrieval.
+
+5. **Advanced Caching**: Implement a distributed caching system for better performance at scale.
+
+## Conclusion
+
+The architecture of Anveshak balances technical sophistication with simplicity and accessibility. By combining modern NLP techniques with traditional spiritual texts, it creates a bridge between ancient wisdom and contemporary technology, making spiritual knowledge more accessible to seekers around the world.
+
+Anveshak: Spirituality Q&A acknowledges and honors Saints, Sages, Siddhas, Yogis, Sadhus, Rishis, Gurus, Mystics, and Spiritual Masters from all backgrounds, genders, traditions, and walks of life, understanding that wisdom transcends all such distinctions. Its focused approach on providing concise, direct answers maintains the essence of spiritual teaching while embracing modern technological capabilities.

docs/changelog-doc.md

@@ -0,0 +1,53 @@
+# Changelog
+
+All notable changes to Anveshak: Spirituality Q&A will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.0] - 2025-04-01
+
+### Added
+- Initial release of Anveshak: Spirituality Q&A
+- Core RAG functionality with E5-large-v2 embedding model
+- FAISS index for efficient text retrieval
+- Integration with OpenAI API for answer generation
+- Streamlit-based user interface
+- Caching mechanisms for improved performance
+- Support for customizable number of sources and word limits
+- Pre-selected common spiritual questions
+- Comprehensive acknowledgment of sources and publishers
+- Detailed documentation
+
+### Technical Features
+- Google Cloud Storage integration for data storage
+- Authentication handling for GCP and OpenAI
+- Memory optimization for resource-constrained environments
+- Multi-page Streamlit application structure
+- Custom CSS styling for enhanced user experience
+- Privacy protection with no user data storage
+- Concise answer generation system
+- Recognition of Saints and Spiritual Masters of all backgrounds and traditions
+
+## Future Roadmap
+
+### Planned for v1.1.0
+- Multi-language support (Sanskrit, Hindi, Bengali, Tamil, and more)
+- User feedback collection for answer quality
+- Enhanced answer relevance with hybrid retrieval methods
+- Additional spiritual texts from diverse traditions
+- Improved citation formatting with page numbers where available
+
+### Planned for v1.2.0
+- Self-hosted open-source LLM integration
+- Advanced visualization of concept relationships
+- Search functionality for specific texts or authors
+- Audio output for visually impaired users
+- Mobile-optimized interface
+
+### Planned for v2.0.0
+- Meditation timer and guide integration
+- Personalized learning paths based on user interests (implemented with privacy-preserving approaches like client-side storage, session-based preferences, or explicit opt-in)
+- Interactive glossary of spiritual terms
+- Spiritual practice guide with scheduler and tracker
+- Community features for discussion and shared learning

docs/configuration-doc.md

@@ -0,0 +1,597 @@
+# Configuration Guide
+
+This document provides detailed instructions for configuring and deploying Anveshak: Spirituality Q&A, covering environment setup, authentication, customization options, and deployment strategies.
+
+## Environment Configuration
+
+### Configuration Parameters
+
+Anveshak: Spirituality Q&A uses the following configuration parameters, which can be set through environment variables or Hugging Face Spaces secrets:
+
+| Parameter | Description | Example Value |
+|-----------|-------------|---------------|
+| `BUCKET_NAME_GCS` | GCS bucket name for data storage | `"your-bucket-name"` |
+| `METADATA_PATH_GCS` | Path to metadata file in GCS | `"metadata/metadata.jsonl"` |
+| `EMBEDDINGS_PATH_GCS` | Path to embeddings file in GCS | `"processed/embeddings/all_embeddings.npy"` |
+| `INDICES_PATH_GCS` | Path to FAISS index in GCS | `"processed/indices/faiss_index.faiss"` |
+| `CHUNKS_PATH_GCS` | Path to text chunks file in GCS | `"processed/chunks/text_chunks.txt"` |
+| `RAW_TEXTS_UPLOADED_PATH_GCS` | Path to uploaded raw texts in GCS | `"raw-texts/uploaded"` |
+| `RAW_TEXTS_DOWNLOADED_PATH_GCS` | Path to downloaded raw texts in GCS | `"raw-texts/downloaded/"` |
+| `CLEANED_TEXTS_PATH_GCS` | Path to cleaned texts in GCS | `"cleaned-texts/"` |
+| `EMBEDDING_MODEL` | Hugging Face model ID for embeddings | `"intfloat/e5-large-v2"` |
+| `LLM_MODEL` | OpenAI model for answer generation | `"gpt-3.5-turbo"` |
+| `OPENAI_API_KEY` | OpenAI API key | `"sk-..."` |
+| `GCP_CREDENTIALS` | GCP service account credentials (JSON) | `{"type":"service_account",...}` |
+
+### Streamlit Secrets Configuration (Optional)
+
+If developing locally with Streamlit, you can create a `.streamlit/secrets.toml` file with the following structure:
+
+```toml
+# GCS Configuration
+BUCKET_NAME_GCS = "your-bucket-name"
+METADATA_PATH_GCS = "metadata/metadata.jsonl"
+EMBEDDINGS_PATH_GCS = "processed/embeddings/all_embeddings.npy"
+INDICES_PATH_GCS = "processed/indices/faiss_index.faiss"
+CHUNKS_PATH_GCS = "processed/chunks/text_chunks.txt"
+RAW_TEXTS_UPLOADED_PATH_GCS = "raw-texts/uploaded"
+RAW_TEXTS_DOWNLOADED_PATH_GCS = "raw-texts/downloaded/"
+CLEANED_TEXTS_PATH_GCS = "cleaned-texts/"
+EMBEDDING_MODEL = "intfloat/e5-large-v2"
+LLM_MODEL = "gpt-3.5-turbo"
+
+# OpenAI API Configuration
+openai_api_key = "your-openai-api-key"
+
+# GCP Service Account Credentials (JSON format)
+[gcp_credentials]
+type = "service_account"
+project_id = "your-project-id"
+private_key_id = "your-private-key-id"
+private_key = "your-private-key"
+client_email = "your-client-email"
+client_id = "your-client-id"
+auth_uri = "https://accounts.google.com/o/oauth2/auth"
+token_uri = "https://oauth2.googleapis.com/token"
+auth_provider_x509_cert_url = "https://www.googleapis.com/oauth2/v1/certs"
+client_x509_cert_url = "your-client-cert-url"
+```
+
+### Environment Variables for Alternative Deployments
+
+For deployments that support environment variables (like Heroku or Docker), you can use the following environment variables:
+
+```bash
+# GCS Configuration
+export BUCKET_NAME_GCS="your-bucket-name"
+export METADATA_PATH_GCS="metadata/metadata.jsonl"
+export EMBEDDINGS_PATH_GCS="processed/embeddings/all_embeddings.npy"
+export INDICES_PATH_GCS="processed/indices/faiss_index.faiss"
+export CHUNKS_PATH_GCS="processed/chunks/text_chunks.txt"
+export RAW_TEXTS_UPLOADED_PATH_GCS="raw-texts/uploaded"
+export RAW_TEXTS_DOWNLOADED_PATH_GCS="raw-texts/downloaded/"
+export CLEANED_TEXTS_PATH_GCS="cleaned-texts/"
+export EMBEDDING_MODEL="intfloat/e5-large-v2"
+export LLM_MODEL="gpt-3.5-turbo"
+
+# OpenAI API Configuration
+export OPENAI_API_KEY="your-openai-api-key"
+
+# GCP Service Account (as a JSON string)
+export GCP_CREDENTIALS='{"type":"service_account","project_id":"your-project-id",...}'
+```
+
+## Authentication Setup
+
+### Google Cloud Storage (GCS) Authentication
+
+Anveshak: Spirituality Q&A supports multiple methods for authenticating with GCS:
+
+#### Setting Up a GCP Service Account (Required)
+
+Before configuring authentication methods, you'll need to create a Google Cloud Platform (GCP) service account:
+
+1. **Create a GCP project** (if you don't already have one):
+   - Go to the [Google Cloud Console](https://console.cloud.google.com/)
+   - Click on "Select a project" at the top right and then "New Project"
+   - Enter a project name and click "Create"
+
+2. **Enable the Cloud Storage API**:
+   - Go to "APIs & Services" > "Library" in the left sidebar
+   - Search for "Cloud Storage"
+   - Click on "Cloud Storage API" and then "Enable"
+
+3. **Create a service account**:
+   - Go to "IAM & Admin" > "Service Accounts" in the left sidebar
+   - Click "Create Service Account"
+   - Enter a service account name and description
+   - Click "Create and Continue"
+
+4. **Assign roles to the service account**:
+   - Add the "Storage Object Admin" role for access to GCS objects
+   - Add the "Viewer" role for basic read permissions
+   - Click "Continue" and then "Done"
+
+5. **Create and download service account key**:
+   - Find your new service account in the list and click on it
+   - Go to the "Keys" tab
+   - Click "Add Key" > "Create new key"
+   - Choose "JSON" as the key type
+   - Click "Create" to download the key file (This is your GCP credentials JSON file)
+
+6. **Create a GCS bucket**:
+   - Go to "Cloud Storage" > "Buckets" in the left sidebar
+   - Click "Create"
+   - Enter a globally unique bucket name
+   - Choose your settings for location, class, and access control
+   - Click "Create"
+
+Once you have created your service account and GCS bucket, you can use any of the following authentication methods:
+
+#### Option 1: HF Spaces Environment Variable (Recommended Production Method)
+
+For Hugging Face Spaces, set the `GCP_CREDENTIALS` environment variable in the Spaces UI:
+
+1. Go to your Space settings
+2. Under "Repository secrets"
+3. Add a new secret with name `GCP_CREDENTIALS` and value containing your JSON credentials
+
+#### Option 2: Local Development with Application Default Credentials
+
+For local development, you can use Application Default Credentials:
+
+```bash
+# Export path to your service account key file
+export GOOGLE_APPLICATION_CREDENTIALS="/path/to/your-service-account-file.json"
+```
+
+#### Option 3: Streamlit Secrets
+
+Add your service account credentials to the `.streamlit/secrets.toml` file as shown in the example above.
+
+The authentication logic is handled by the `setup_gcp_auth()` function in `utils.py`:
+
+```python
+def setup_gcp_auth():
+    """
+    Setup Google Cloud Platform (GCP) authentication using various methods.
+    
+    This function tries multiple authentication methods in order of preference:
+    1. HF Spaces environment variable (GCP_CREDENTIALS) - primary production method
+    2. Local environment variable pointing to credentials file (GOOGLE_APPLICATION_CREDENTIALS)
+    3. Streamlit secrets (gcp_credentials)
+    
+    Note: In production, credentials are stored exclusively in HF Spaces secrets.
+    """
+    try:
+        # Option 1: HF Spaces environment variable
+        if "GCP_CREDENTIALS" in os.environ:
+            gcp_credentials = json.loads(os.getenv("GCP_CREDENTIALS"))
+            print("✅ Using GCP credentials from HF Spaces environment variable")
+            credentials = service_account.Credentials.from_service_account_info(gcp_credentials)
+            return credentials
+            
+        # Option 2: Local environment variable pointing to file
+        elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
+            credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
+            print(f"✅ Using GCP credentials from file at {credentials_path}")
+            credentials = service_account.Credentials.from_service_account_file(credentials_path)
+            return credentials
+            
+        # Option 3: Streamlit secrets
+        elif "gcp_credentials" in st.secrets:
+            gcp_credentials = st.secrets["gcp_credentials"]
+            
+            # Handle different secret formats
+            if isinstance(gcp_credentials, dict) or hasattr(gcp_credentials, 'to_dict'):
+                # Convert AttrDict to dict if needed
+                if hasattr(gcp_credentials, 'to_dict'):
+                    gcp_credentials = gcp_credentials.to_dict()
+                    
+                print("✅ Using GCP credentials from Streamlit secrets (dict format)")
+                credentials = service_account.Credentials.from_service_account_info(gcp_credentials)
+                return credentials
+            else:
+                # Assume it's a JSON string
+                try:
+                    gcp_credentials_dict = json.loads(gcp_credentials)
+                    print("✅ Using GCP credentials from Streamlit secrets (JSON string)")
+                    credentials = service_account.Credentials.from_service_account_info(gcp_credentials_dict)
+                    return credentials
+                except json.JSONDecodeError:
+                    print("⚠️ GCP credentials in Streamlit secrets is not valid JSON, trying as file path")
+                    if os.path.exists(gcp_credentials):
+                        credentials = service_account.Credentials.from_service_account_file(gcp_credentials)
+                        return credentials
+                    else:
+                        raise ValueError("GCP credentials format not recognized")
+                        
+        else:
+            raise ValueError("No GCP credentials found in environment or Streamlit secrets")
+            
+    except Exception as e:
+        error_msg = f"❌ Authentication error: {str(e)}"
+        print(error_msg)
+        st.error(error_msg)
+        raise
+```
+
+### OpenAI API Authentication
+
+Similarly, OpenAI API authentication can be configured in multiple ways:
+
+#### Option 1: HF Spaces Environment Variable (Recommended Production Method)
+
+Set the `OPENAI_API_KEY` environment variable in the Hugging Face Spaces UI.
+
+#### Option 2: Environment Variables
+
+Set the `OPENAI_API_KEY` environment variable:
+
+```bash
+export OPENAI_API_KEY="your-openai-api-key"
+```
+
+#### Option 3: Streamlit Secrets
+
+Add your OpenAI API key to the `.streamlit/secrets.toml` file:
+
+```toml
+openai_api_key = "your-openai-api-key"
+```
+
+The authentication logic is handled by the `setup_openai_auth()` function in `utils.py`:
+
+```python
+def setup_openai_auth():
+    """
+    Setup OpenAI API authentication using various methods.
+    
+    This function tries multiple authentication methods in order of preference:
+    1. Standard environment variable (OPENAI_API_KEY)
+    2. HF Spaces environment variable (OPENAI_KEY) - primary production method
+    3. Streamlit secrets (openai_api_key)
+    
+    Note: In production, the API key is stored exclusively in HF Spaces secrets.
+    """
+    try:
+        # Option 1: Standard environment variable
+        if "OPENAI_API_KEY" in os.environ:
+            openai.api_key = os.getenv("OPENAI_API_KEY")
+            print("✅ Using OpenAI API key from environment variable")
+            return
+            
+        # Option 2: HF Spaces environment variable with different name
+        elif "OPENAI_KEY" in os.environ:
+            openai.api_key = os.getenv("OPENAI_KEY")
+            print("✅ Using OpenAI API key from HF Spaces environment variable")
+            return
+            
+        # Option 3: Streamlit secrets
+        elif "openai_api_key" in st.secrets:
+            openai.api_key = st.secrets["openai_api_key"]
+            print("✅ Using OpenAI API key from Streamlit secrets")
+            return
+            
+        else:
+            raise ValueError("No OpenAI API key found in environment or Streamlit secrets")
+            
+    except Exception as e:
+        error_msg = f"❌ OpenAI authentication error: {str(e)}"
+        print(error_msg)
+        st.error(error_msg)
+        raise
+```
+
+## Application Customization
+
+### UI Customization
+
+Anveshak's UI can be customized through the CSS in the `app.py` file:
+
+```python
+# Custom CSS
+st.markdown("""
+<style>
+.main-title {
+    font-size: 2.5rem;
+    color: #c0392b;
+    text-align: center;
+    margin-bottom: 1rem;
+}
+.subtitle {
+    font-size: 1.2rem;
+    color: #555;
+    text-align: center;
+    margin-bottom: 1.5rem;
+    font-style: italic;
+}
+/* More CSS rules... */
+</style>
+<div class="main-title">Anveshak</div>
+<div class="subtitle">Spirituality Q&A</div>
+""", unsafe_allow_html=True)
+```
+
+To change the appearance:
+
+1. Modify the CSS variables in the `<style>` tag
+2. Update color schemes, fonts, or layouts as needed
+3. Add new CSS classes for additional UI elements
+
+### Common Questions Configuration
+
+The list of pre-selected common questions can be modified in the `app.py` file:
+
+```python
+# Common spiritual questions for users to select from
+common_questions = [
+    "What is the Atman or the soul?",
+    "Are there rebirths?",
+    "What is Karma?",
+    # Add or modify questions here
+]
+```
+
+### Retrieval Parameters
+
+Two key retrieval parameters can be adjusted by users through the UI:
+
+1. **Number of sources** (`top_k`): Controls how many distinct sources are used for generating answers
+   - Default: 5
+   - Range: 3-10
+   - UI Component: Slider in the main interface
+
+2. **Word limit** (`word_limit`): Controls the maximum length of generated answers
+   - Default: 200
+   - Range: 50-500
+   - UI Component: Slider in the main interface
+
+These parameters are implemented in the Streamlit UI:
+
+```python
+# Sliders for customization
+col1, col2 = st.columns(2)
+with col1:
+    top_k = st.slider("Number of sources:", 3, 10, 5)
+with col2:
+    word_limit = st.slider("Word limit:", 50, 500, 200)
+```
+
+## Deployment Options
+
+### Recommended: Hugging Face Spaces Deployment
+
+The recommended and tested deployment method for Anveshak: Spirituality Q&A is Hugging Face Spaces, which provides the necessary resources for running the application efficiently.
+
+To deploy on Hugging Face Spaces:
+
+1. Fork the repository to your GitHub account
+
+2. Create a new Space on Hugging Face:
+   - Go to [huggingface.co/spaces](https://huggingface.co/spaces)
+   - Click "Create new Space"
+   - Select "Streamlit" as the SDK
+   - Connect your GitHub repository
+
+3. Configure secrets in the Hugging Face UI:
+   - Go to your Space settings
+   - Under "Repository secrets"
+   - Add each of the following secrets:
+     - `OPENAI_API_KEY`
+     - `GCP_CREDENTIALS` (the entire JSON as a string)
+     - `BUCKET_NAME_GCS`
+     - `LLM_MODEL`
+     - `METADATA_PATH_GCS`
+     - `RAW_TEXTS_UPLOADED_PATH_GCS`
+     - `RAW_TEXTS_DOWNLOADED_PATH_GCS`
+     - `CLEANED_TEXTS_PATH_GCS`
+     - `EMBEDDINGS_PATH_GCS`
+     - `INDICES_PATH_GCS`
+     - `CHUNKS_PATH_GCS`
+     - `EMBEDDING_MODEL`
+
+4. The app should automatically deploy. If needed, manually trigger a rebuild from the Spaces UI.
+
+### Local Development (Not Recommended)
+
+**Important Note**: Running Anveshak: Spirituality Q&A locally requires above 16GB of RAM due to the embedding model. Most standard laptops will experience crashes during model loading. Hugging Face Spaces deployment is strongly recommended.
+
+If you still want to run it locally for development purposes:
+
+1. Clone the repository
+   ```bash
+   git clone https://github.com/YourUsername/anveshak.git
+   cd anveshak
+   ```
+
+2. Install dependencies
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+3. Create the `.streamlit/secrets.toml` file as described above
+
+4. Run the application
+   ```bash
+   streamlit run app.py
+   ```
+
+### Alternative: Docker Deployment
+
+For containerized deployment (not tested in production):
+
+1. Create a `Dockerfile`:
+
+```dockerfile
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . .
+
+EXPOSE 8501
+
+CMD ["streamlit", "run", "app.py"]
+```
+
+2. Build the Docker image:
+```bash
+docker build -t anveshak .
+```
+
+3. Run the container:
+```bash
+docker run -p 8501:8501 \
+  -e BUCKET_NAME_GCS=your-bucket-name \
+  -e METADATA_PATH_GCS=metadata/metadata.jsonl \
+  -e EMBEDDINGS_PATH_GCS=processed/embeddings/all_embeddings.npy \
+  -e INDICES_PATH_GCS=processed/indices/faiss_index.faiss \
+  -e CHUNKS_PATH_GCS=processed/chunks/text_chunks.txt \
+  -e RAW_TEXTS_UPLOADED_PATH_GCS=raw-texts/uploaded \
+  -e RAW_TEXTS_DOWNLOADED_PATH_GCS=raw-texts/downloaded/ \
+  -e CLEANED_TEXTS_PATH_GCS=cleaned-texts/ \
+  -e EMBEDDING_MODEL=intfloat/e5-large-v2 \
+  -e LLM_MODEL=gpt-3.5-turbo \
+  -e OPENAI_API_KEY=your-openai-api-key \
+  -e GCP_CREDENTIALS='{"type":"service_account",...}' \
+  anveshak
+```
+
+## Performance Tuning
+
+### Caching Configuration
+
+Anveshak: Spirituality Q&A uses Streamlit's caching mechanisms to optimize performance:
+
+#### Resource Caching
+Used for loading models and data files that remain constant:
+
+```python
+@st.cache_resource(show_spinner=False)
+def cached_load_model():
+    # Load embedding model once and cache it
+```
+
+This cache persists for the lifetime of the application.
+
+#### Data Caching
+Used for caching query results with a time-to-live (TTL):
+
+```python
+@st.cache_data(ttl=3600, show_spinner=False)
+def cached_process_query(query, top_k=5, word_limit=100):
+    # Cache query results for an hour
+```
+
+The TTL (3600 seconds = 1 hour) can be adjusted based on your needs.
+
+### Memory Optimization
+
+For deployments with limited memory:
+
+1. **Force CPU Usage**: Anveshak already forces CPU usage for the embedding model to avoid GPU memory issues:
+   ```python
+   os.environ["CUDA_VISIBLE_DEVICES"] = ""
+   ```
+
+2. **Adjust Batch Size**: If you're recreating the embeddings, consider reducing the batch size:
+   ```python
+   def create_embeddings(text_chunks, batch_size=16):  # Reduced from 32
+   ```
+
+3. **Garbage Collection**: Anveshak performs explicit garbage collection after operations:
+   ```python
+   del outputs, inputs
+   gc.collect()
+   ```
+
+## Troubleshooting
+
+### Common Issues
+
+#### Authentication Errors
+
+**Symptom**: Error message about invalid credentials or permission denied.
+
+**Solution**:
+1. Verify that your service account has the correct permissions (Storage Object Admin)
+2. Check that your API keys are correctly formatted and not expired
+3. Ensure that your GCP credentials JSON is valid and properly formatted
+
+#### Missing Files
+
+**Symptom**: Error about missing files or "File not found" when accessing GCS.
+
+**Solution**:
+1. Verify the correct bucket name and file paths in your configuration
+2. Check that all required files exist in your GCS bucket
+3. Ensure your service account has access to the specified bucket
+
+#### Memory Issues
+
+**Symptom**: Application crashes with out-of-memory errors.
+
+**Solution**:
+1. Increase the memory allocation for your deployment (if possible)
+2. Ensure that `os.environ["CUDA_VISIBLE_DEVICES"] = ""` is set to force CPU usage
+3. Implement additional garbage collection calls in high-memory operations
+
+#### OpenAI API Rate Limits
+
+**Symptom**: Errors about rate limits or exceeding quotas with OpenAI.
+
+**Solution**:
+1. Implement retry logic with exponential backoff
+2. Consider using a paid tier OpenAI account with higher rate limits
+3. Add caching to reduce the number of API calls
+
+### Logs and Debugging
+
+Anveshak includes comprehensive logging:
+
+```python
+print(f"✅ Model loaded successfully (cached)")
+print(f"❌ Error loading model: {str(e)}")
+```
+
+To enable more detailed logging, you can use Streamlit's built-in logging configuration:
+
+```python
+import logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Then use logger instead of print
+logger.info("Model loaded successfully")
+logger.error(f"Error loading model: {str(e)}")
+```
+
+## Special Considerations
+
+### Privacy
+
+Anveshak: Spirituality Q&A is designed to not save or store any user queries or data. This is important for spiritual questions, which may be of a personal nature. No additional configuration is needed for this - the application simply does not implement any data storage functionality.
+
+### Language Support
+
+Currently, Anveshak is only available in English. This is a known limitation of the current implementation. Future versions may include support for Sanskrit, Hindi, Bengali, Tamil, and other Indian languages.
+
+### Concise Answers
+
+Anveshak generates concise answers rather than lengthy explanations. This is by design, to respect both copyright constraints and the nature of spiritual wisdom, which often benefits from clarity and simplicity.
+
+## Conclusion
+
+This configuration guide provides all the necessary information to set up, customize, and deploy Anveshak: Spirituality Q&A. By following these instructions, you should be able to:
+
+1. Configure the necessary authentication for GCS and OpenAI
+2. Customize Anveshak's appearance and behavior
+3. Deploy the application on Hugging Face Spaces (recommended) or other platforms
+4. Optimize performance for your specific use case
+5. Troubleshoot common issues
+
+The flexibility of the configuration options allows you to adapt the application to different deployment environments while maintaining the core functionality of providing spiritually informed answers based on traditional texts from diverse traditions and teachers of all backgrounds.

docs/data-handling-doc.md

@@ -0,0 +1,687 @@
+# Data Handling Explanation
+
+This document explains how data is processed, stored, and handled in Anveshak: Spirituality Q&A, with special attention to ethical considerations and copyright respect.
+
+## Data Sources
+
+### Text Corpus Overview
+
+Anveshak: Spirituality Q&A uses approximately 133 digitized spiritual texts sourced from freely available resources. These texts include:
+
+- Ancient sacred literature (Vedas, Upanishads, Puranas, Sutras, Dharmaśāstras, and Agamas)
+- Classical Indian texts (The Bhagavad Gita, The Śrīmad Bhāgavatam, and others)
+- Indian historical texts (The Mahabharata and The Ramayana)
+- Teachings of revered Saints, Sages, Siddhas, Yogis, Sadhus, Rishis, Gurus, Mystics, and Spiritual Masters of all genders, backgrounds, traditions, and walks of life
+
+As stated in app.py:
+
+> "Anveshak draws from a rich tapestry of spiritual wisdom found in classical Indian texts, philosophical treatises, and the teachings of revered Saints, Sages, Siddhas, Yogis, Sadhus, Rishis, Gurus, Mystics, and Spiritual Masters across centuries. The knowledge presented here spans multiple traditions, schools of thought, and spiritual lineages that have flourished in the Indian subcontinent and beyond."
+
+### Ethical Sourcing
+
+All texts included in Anveshak meet the following criteria:
+
+1. **Public availability**: All texts were freely available from sources like archive.org
+2. **Educational use**: Texts are used solely for educational purposes
+3. **Proper attribution**: All sources are credited with author and publisher information
+4. **Respect for copyright**: Implementation of word limits and other copyright-respecting measures
+
+As mentioned in app.py:
+
+> "Note that the sources consist of about 133 digitized texts, all of which were freely available over the internet (on sites like archive.org). Many of the texts are English translations of original (and in some cases, ancient) sacred and spiritual texts. All of the copyrights belong to the respective authors and publishers and we bow down in gratitude to their selfless work. Anveshak merely re-presents the ocean of spiritual knowledge and wisdom contained in the original works with relevant citations in a limited number of words."
+
+## Data Processing Pipeline
+
+### 1. Data Collection
+
+The data collection process involves two methods as implemented in preprocessing.py:
+
+#### Manual Upload
+Texts are manually uploaded to Google Cloud Storage (GCS) through a preprocessing script:
+
+```python
+def upload_files_to_colab():
+    """Upload raw text files and metadata from local machine to Colab."""
+    # First, upload text files
+    print("Step 1: Please upload your text files...")
+    uploaded_text_files = files.upload()  # This will prompt the user to upload files
+
+    # Create directory structure if it doesn't exist
+    os.makedirs(LOCAL_RAW_TEXTS_FOLDER, exist_ok=True)
+
+    # Move uploaded text files to the raw-texts folder
+    for filename, content in uploaded_text_files.items():
+        if filename.endswith(".txt"):
+            with open(os.path.join(LOCAL_RAW_TEXTS_FOLDER, filename), "wb") as f:
+                f.write(content)
+            print(f"✅ Saved {filename} to {LOCAL_RAW_TEXTS_FOLDER}")
+```
+
+#### Web Downloading
+Some texts are automatically downloaded from URLs listed in the metadata file:
+
+```python
+def download_text_files():
+    """Fetch metadata, filter unuploaded files, and download text files."""
+    metadata = fetch_metadata_from_gcs()
+    # Filter entries where Uploaded is False
+    files_to_download = [item for item in metadata if item["Uploaded"] == False]
+    
+    # Process only necessary files
+    for item in files_to_download:
+        name, author, url = item["Title"], item["Author"], item["URL"]
+        if url.lower() == "not available":
+            print(f"❌ Skipping {name} - No URL available.")
+            continue
+
+        try:
+            response = requests.get(url)
+            if response.status_code == 200:
+                raw_text = response.text
+                filename = "{}.txt".format(name.replace(" ", "_"))
+                # Save to local first
+                local_path = f"/tmp/{filename}"
+                with open(local_path, "w", encoding="utf-8") as file:
+                    file.write(raw_text)
+                # Upload to GCS
+                gcs_path = f"{RAW_TEXTS_DOWNLOADED_PATH_GCS}{filename}"
+                upload_to_gcs(local_path, gcs_path)
+                print(f"✅ Downloaded & uploaded: {filename} ({len(raw_text.split())} words)")
+            else:
+                print(f"❌ Failed to download {name}: {url} (Status {response.status_code})")
+        except Exception as e:
+            print(f"❌ Error processing {name}: {e}")
+```
+
+### 2. Text Cleaning
+
+Raw texts often contain HTML tags, OCR errors, and formatting issues. The cleaning process removes these artifacts using the exact implementation from preprocessing.py:
+
+```python
+def rigorous_clean_text(text):
+    """
+    Clean text by removing metadata, junk text, and formatting issues.
+
+    This function:
+    1. Removes HTML tags using BeautifulSoup
+    2. Removes URLs and standalone numbers
+    3. Removes all-caps OCR noise words
+    4. Deduplicates adjacent identical lines
+    5. Normalizes Unicode characters
+    6. Standardizes whitespace and newlines
+
+    Args:
+        text (str): The raw text to clean
+
+    Returns:
+        str: The cleaned text
+    """
+    text = BeautifulSoup(text, "html.parser").get_text()
+    text = re.sub(r"https?:\/\/\S+", "", text)  # Remove links
+    text = re.sub(r"\b\d+\b", "", text)  # Remove standalone numbers
+    text = re.sub(r"\b[A-Z]{5,}\b", "", text)  # Remove all-caps OCR noise words
+    lines = text.split("\n")
+    cleaned_lines = []
+    last_line = None
+
+    for line in lines:
+        line = line.strip()
+        if line and line != last_line:
+            cleaned_lines.append(line)
+            last_line = line
+
+    text = "\n".join(cleaned_lines)
+    text = unicodedata.normalize("NFKD", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    text = re.sub(r"\n{2,}", "\n", text)
+    return text
+```
+
+The cleaning process:
+- Removes HTML tags using BeautifulSoup
+- Eliminates URLs and standalone numbers
+- Removes all-caps OCR noise words (common in digitized texts)
+- Deduplicates adjacent identical lines
+- Normalizes Unicode characters
+- Standardizes whitespace and newlines
+
+### 3. Text Chunking
+
+Clean texts are split into smaller, manageable chunks for processing using the exact implementation from preprocessing.py:
+
+```python
+def chunk_text(text, chunk_size=500, overlap=50):
+    """
+    Split text into smaller, overlapping chunks for better retrieval.
+    
+    Args:
+        text (str): The text to chunk
+        chunk_size (int): Maximum number of words per chunk
+        overlap (int): Number of words to overlap between chunks
+    
+    Returns:
+        list: List of text chunks
+    """
+    words = text.split()
+    chunks = []
+    i = 0
+
+    while i < len(words):
+        chunk = " ".join(words[i:i + chunk_size])
+        chunks.append(chunk)
+        i += chunk_size - overlap
+
+    return chunks
+```
+
+Chunking characteristics:
+- **Chunk size**: 500 words per chunk, balancing context and retrieval precision
+- **Overlap**: 50-word overlap between chunks to maintain context across chunk boundaries
+- **Context preservation**: Ensures that passages aren't arbitrarily cut in the middle of important concepts
+
+### 4. Text Embedding
+
+Chunks are converted to vector embeddings using the E5-large-v2 model with the actual implementation from preprocessing.py:
+
+```python
+def create_embeddings(text_chunks, batch_size=32):
+    """
+    Generate embeddings for the given chunks of text using the specified embedding model.
+    
+    This function:
+    1. Uses SentenceTransformer to load the embedding model
+    2. Prefixes each chunk with "passage:" as required by the E5 model
+    3. Processes chunks in batches to manage memory usage
+    4. Normalizes embeddings for cosine similarity search
+    
+    Args:
+        text_chunks (list): List of text chunks to embed
+        batch_size (int): Number of chunks to process at once
+    
+    Returns:
+        numpy.ndarray: Matrix of embeddings, one per text chunk
+    """
+    # Load the model with GPU optimization
+    model = SentenceTransformer(EMBEDDING_MODEL)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = model.to(device)
+    print(f"🚀 Using device for embeddings: {device}")
+
+    prefixed_chunks = [f"passage: {text}" for text in text_chunks]
+    all_embeddings = []
+
+    for i in range(0, len(prefixed_chunks), batch_size):
+        batch = prefixed_chunks[i:i+batch_size]
+        # Move batch to GPU (if available) for faster processing
+        with torch.no_grad():
+            batch_embeddings = model.encode(batch, convert_to_numpy=True, normalize_embeddings=True)
+        all_embeddings.append(batch_embeddings)
+
+        if (i + batch_size) % 100 == 0 or (i + batch_size) >= len(prefixed_chunks):
+            print(f"📌 Processed {i + min(batch_size, len(prefixed_chunks) - i)}/{len(prefixed_chunks)} documents")
+
+    return np.vstack(all_embeddings).astype("float32")
+```
+
+Embedding process details:
+- **Model**: E5-large-v2, a state-of-the-art embedding model for retrieval tasks
+- **Prefix**: "passage:" prefix is added to each chunk for optimal embedding
+- **Batching**: Processing in batches of 32 for memory efficiency
+- **Normalization**: Embeddings are normalized for cosine similarity search
+- **Output**: Each text chunk becomes a 1024-dimensional vector
+
+### 5. FAISS Index Creation
+
+Embeddings are stored in a Facebook AI Similarity Search (FAISS) index for efficient similarity search:
+
+```python
+# Build FAISS index
+dimension = all_embeddings.shape[1]
+index = faiss.IndexFlatIP(dimension)
+index.add(all_embeddings)
+```
+
+FAISS index characteristics:
+- **Index type**: IndexFlatIP (Inner Product) for cosine similarity search
+- **Exact search**: Uses exact search rather than approximate for maximum accuracy
+- **Dimension**: 1024-dimensional vectors from the E5-large-v2 model
+
+### 6. Metadata Management
+
+The system maintains metadata for each text to provide proper citations, using the implementation from rag_engine.py:
+
+```python
+def fetch_metadata_from_gcs():
+    """
+    Fetch metadata.jsonl from GCS and return as a list of dictionaries.
+    
+    Each dictionary represents a text entry with metadata like title, author, etc.
+    
+    Returns:
+        list: List of dictionaries containing metadata for each text
+    """
+    blob = bucket.blob(METADATA_PATH_GCS)
+    # Download metadata file
+    metadata_jsonl = blob.download_as_text()
+    # Parse JSONL
+    metadata = [json.loads(line) for line in metadata_jsonl.splitlines()]
+    return metadata
+```
+
+Metadata structure (JSONL format):
+```json
+{"Title": "Bhagavad Gita", "Author": "Vyasa", "Publisher": "Gita Press, Gorakhpur, India", "URL": "https://archive.org/details/bhagavad-gita", "Uploaded": true}
+{"Title": "Yoga Sutras", "Author": "Patanjali", "Publisher": "DIVINE LIFE SOCIETY", "URL": "https://archive.org/details/yoga-sutras", "Uploaded": true}
+```
+
+## Data Storage Architecture
+
+### Google Cloud Storage Structure
+
+Anveshak: Spirituality Q&A uses Google Cloud Storage (GCS) as its primary data store, organized as follows:
+
+```
+bucket_name/
+├── metadata/
+│   └── metadata.jsonl           # Metadata for all texts
+├── raw-texts/
+│   ├── uploaded/                # Manually uploaded texts
+│   └── downloaded/              # Automatically downloaded texts
+├── cleaned-texts/               # Cleaned versions of all texts
+└── processed/
+    ├── embeddings/
+    │   └── all_embeddings.npy   # Numpy array of embeddings
+    ├── indices/
+    │   └── faiss_index.faiss    # FAISS index file
+    └── chunks/
+        └── text_chunks.txt      # Text chunks with metadata
+```
+
+### Local Caching
+
+For deployment on Hugging Face Spaces, essential files are downloaded to local storage using the implementation from rag_engine.py:
+
+```python
+# Local Paths
+local_embeddings_file = "all_embeddings.npy"
+local_faiss_index_file = "faiss_index.faiss"
+local_text_chunks_file = "text_chunks.txt"
+local_metadata_file = "metadata.jsonl"
+```
+
+These files are loaded with caching to improve performance, using the actual implementation from rag_engine.py:
+
+```python
+@st.cache_resource(show_spinner=False)
+def cached_load_data_files():
+    """
+    Cached version of load_data_files() for FAISS index, text chunks, and metadata.
+    
+    This function loads:
+    - FAISS index for vector similarity search
+    - Text chunks containing the original spiritual text passages
+    - Metadata dictionary with publication and author information
+    
+    All files are downloaded from Google Cloud Storage if not already present locally.
+    
+    Returns:
+        tuple: (faiss_index, text_chunks, metadata_dict) or (None, None, None) if loading fails
+    """
+    # Initialize GCP and OpenAI clients
+    bucket = setup_gcp_client()
+    openai_initialized = setup_openai_client()
+    
+    if not bucket or not openai_initialized:
+        print("Failed to initialize required services")
+        return None, None, None
+    
+    # Get GCS paths from secrets - required
+    try:
+        metadata_file_gcs = st.secrets["METADATA_PATH_GCS"]
+        embeddings_file_gcs = st.secrets["EMBEDDINGS_PATH_GCS"]
+        faiss_index_file_gcs = st.secrets["INDICES_PATH_GCS"]
+        text_chunks_file_gcs = st.secrets["CHUNKS_PATH_GCS"]
+    except KeyError as e:
+        print(f"❌ Error: Required GCS path not found in secrets: {e}")
+        return None, None, None
+    
+    # Download necessary files if not already present locally
+    success = True
+    success &= download_file_from_gcs(bucket, faiss_index_file_gcs, local_faiss_index_file)
+    success &= download_file_from_gcs(bucket, text_chunks_file_gcs, local_text_chunks_file)
+    success &= download_file_from_gcs(bucket, metadata_file_gcs, local_metadata_file)
+    
+    if not success:
+        print("Failed to download required files")
+        return None, None, None
+    
+    # Load FAISS index, text chunks, and metadata
+    try:
+        faiss_index = faiss.read_index(local_faiss_index_file)
+    except Exception as e:
+        print(f"❌ Error loading FAISS index: {str(e)}")
+        return None, None, None
+    
+    # Load text chunks
+    try:
+        text_chunks = {}  # Mapping: ID -> (Title, Author, Text)
+        with open(local_text_chunks_file, "r", encoding="utf-8") as f:
+            for line in f:
+                parts = line.strip().split("\t")
+                if len(parts) == 4:
+                    text_chunks[int(parts[0])] = (parts[1], parts[2], parts[3])
+    except Exception as e:
+        print(f"❌ Error loading text chunks: {str(e)}")
+        return None, None, None
+    
+    # Load metadata
+    try:
+        metadata_dict = {}
+        with open(local_metadata_file, "r", encoding="utf-8") as f:
+            for line in f:
+                item = json.loads(line)
+                metadata_dict[item["Title"]] = item
+    except Exception as e:
+        print(f"❌ Error loading metadata: {str(e)}")
+        return None, None, None
+    
+    print(f"✅ Data loaded successfully (cached): {len(text_chunks)} passages available")
+    return faiss_index, text_chunks, metadata_dict
+```
+
+## Data Access During Query Processing
+
+### Query Embedding
+
+User queries are embedded using the same model as the text corpus, with the actual implementation from rag_engine.py:
+
+```python
+def get_embedding(text):
+    """
+    Generate embeddings for a text query using the cached model.
+    
+    Uses an in-memory cache to avoid redundant embedding generation for repeated queries.
+    Properly prefixes inputs with "query:" or "passage:" as required by the E5 model.
+    
+    Args:
+        text (str): The query text to embed
+        
+    Returns:
+        numpy.ndarray: The embedding vector or a zero vector if embedding fails
+    """
+    if text in query_embedding_cache:
+        return query_embedding_cache[text]
+
+    try:
+        tokenizer, model = cached_load_model()
+        if model is None:
+            print("Model is None, returning zero embedding")
+            return np.zeros((1, 384), dtype=np.float32)
+            
+        # Format input based on text length
+        # For E5 models, "query:" prefix is for questions, "passage:" for documents
+        input_text = f"query: {text}" if len(text) < 512 else f"passage: {text}"
+        inputs = tokenizer(
+            input_text,
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
+            max_length=512,
+            return_attention_mask=True
+        )
+        with torch.no_grad():
+            outputs = model(**inputs)
+            embeddings = average_pool(outputs.last_hidden_state, inputs['attention_mask'])
+            embeddings = nn.functional.normalize(embeddings, p=2, dim=1)
+            embeddings = embeddings.detach().cpu().numpy()
+        del outputs, inputs
+        gc.collect()
+        query_embedding_cache[text] = embeddings
+        return embeddings
+    except Exception as e:
+        print(f"❌ Embedding error: {str(e)}")
+        return np.zeros((1, 384), dtype=np.float32)
+```
+
+Note the use of:
+- **Query prefix**: "query:" is added to distinguish query embeddings from passage embeddings
+- **Truncation**: Queries are truncated to 512 tokens if necessary
+- **Memory management**: Tensors are detached and moved to CPU after computation
+- **Caching**: Query embeddings are cached to avoid redundant computation
+
+### Passage Retrieval
+
+The system retrieves relevant passages based on query embedding similarity using the implementation from rag_engine.py:
+
+```python
+def retrieve_passages(query, faiss_index, text_chunks, metadata_dict, top_k=5, similarity_threshold=0.5):
+    """
+    Retrieve the most relevant passages for a given spiritual query.
+    
+    This function:
+    1. Embeds the user query using the same model used for text chunks
+    2. Finds similar passages using the FAISS index with cosine similarity
+    3. Filters results based on similarity threshold to ensure relevance
+    4. Enriches results with metadata (title, author, publisher)
+    5. Ensures passage diversity by including only one passage per source title
+    
+    Args:
+        query (str): The user's spiritual question
+        faiss_index: FAISS index containing passage embeddings
+        text_chunks (dict): Dictionary mapping IDs to text chunks and metadata
+        metadata_dict (dict): Dictionary containing publication information
+        top_k (int): Maximum number of passages to retrieve
+        similarity_threshold (float): Minimum similarity score (0.0-1.0) for retrieved passages
+        
+    Returns:
+        tuple: (retrieved_passages, retrieved_sources) containing the text and source information
+    """
+    try:
+        print(f"\n🔍 Retrieving passages for query: {query}")
+        query_embedding = get_embedding(query)
+        distances, indices = faiss_index.search(query_embedding, top_k * 2)
+        print(f"Found {len(distances[0])} potential matches")
+        retrieved_passages = []
+        retrieved_sources = []
+        cited_titles = set()
+        for dist, idx in zip(distances[0], indices[0]):
+            print(f"Distance: {dist:.4f}, Index: {idx}")
+            if idx in text_chunks and dist >= similarity_threshold:
+                title_with_txt, author, text = text_chunks[idx]
+                clean_title = title_with_txt.replace(".txt", "") if title_with_txt.endswith(".txt") else title_with_txt
+                clean_title = unicodedata.normalize("NFC", clean_title)
+                if clean_title in cited_titles:
+                    continue  
+                metadata_entry = metadata_dict.get(clean_title, {})
+                author = metadata_entry.get("Author", "Unknown")
+                publisher = metadata_entry.get("Publisher", "Unknown")
+                cited_titles.add(clean_title)
+                retrieved_passages.append(text)
+                retrieved_sources.append((clean_title, author, publisher))
+                if len(retrieved_passages) == top_k:
+                    break
+        print(f"Retrieved {len(retrieved_passages)} passages")
+        return retrieved_passages, retrieved_sources
+    except Exception as e:
+        print(f"❌ Error in retrieve_passages: {str(e)}")
+        return [], []
+```
+
+Important aspects:
+- **Similarity threshold**: Passages must have a similarity score >= 0.5 to be included
+- **Diversity**: Only one passage per source title is included in the results
+- **Metadata enrichment**: Publisher information is added from the metadata
+- **Configurable retrieval**: The `top_k` parameter allows users to adjust how many sources to use
+
+## User Data Privacy
+
+### No Data Collection
+
+Anveshak is designed to respect user privacy by not collecting or storing any user data:
+
+1. **No Query Storage**: User questions are processed in memory and not saved
+2. **No User Identification**: No user accounts or identification is required
+3. **No Analytics**: No usage tracking or analytics are implemented
+4. **No Cookies**: No browser cookies are used to track users
+
+As stated in app.py:
+
+> "We do not save any user data or queries. However, user questions are processed using OpenAI's LLM service to generate responses. While we do not store this information, please be aware that interactions are processed through OpenAI's platform and are subject to their privacy policies and data handling practices."
+
+This privacy-first approach ensures that users can freely explore spiritual questions without concerns about their queries being stored or analyzed.
+
+## Copyright and Ethical Considerations
+
+### Word Limit Implementation
+
+To respect copyright and ensure fair use, answers are limited to a configurable word count using the actual implementation from rag_engine.py:
+
+```python
+def answer_with_llm(query, context=None, word_limit=100):
+    # ... LLM processing ...
+    
+    # Extract and format the answer
+    answer = response.choices[0].message.content.strip()
+    words = answer.split()
+    if len(words) > word_limit:
+        answer = " ".join(words[:word_limit])
+        if not answer.endswith((".", "!", "?")):
+            answer += "."
+            
+    return answer
+```
+
+Users can adjust the word limit from 50 to 500 words, ensuring that responses are:
+- Short enough to respect copyright
+- Long enough to provide meaningful information
+- Always properly cited to the original source
+
+### Citation Format
+
+Every answer includes citations to the original sources using the implementation from rag_engine.py:
+
+```python
+def format_citations(sources):
+    """
+    Format citations for display to the user.
+    
+    Creates properly formatted citations for each source used in generating the answer.
+    Each citation appears on a new line with consistent formatting.
+    
+    Args:
+        sources (list): List of (title, author, publisher) tuples
+        
+    Returns:
+        str: Formatted citations as a string with each citation on a new line
+    """
+    formatted_citations = []
+    for title, author, publisher in sources:
+        if publisher.endswith(('.', '!', '?')):
+            formatted_citations.append(f"📚 {title} by {author}, Published by {publisher}")
+        else:
+            formatted_citations.append(f"📚 {title} by {author}, Published by {publisher}.")
+    return "\n".join(formatted_citations)
+```
+
+Citations include:
+- Book/text title
+- Author name
+- Publisher information
+
+### Acknowledgment of Sources
+
+Anveshak: Spirituality Q&A includes dedicated pages for acknowledging:
+- Publishers of the original texts
+- Saints, Sages, and Spiritual Masters whose teachings are referenced
+- The origins and traditions of the spiritual texts
+
+A thank-you note is also prominently featured on the main page, as shown in app.py:
+
+```python
+st.markdown('<div class="acknowledgment-header">A Heartfelt Thank You</div>', unsafe_allow_html=True)
+st.markdown("""
+It is believed that one cannot be in a spiritual path without the will of the Lord. One need not be a believer or a non-believer, merely proceeding to thoughtlessness and observation is enough to evolve and shape perspectives. But that happens through grace. It is believed that without the will of the Lord, one cannot be blessed by real Saints, and without the will of the Saints, one cannot get close to them or God.
+
+Therefore, with deepest reverence, we express our gratitude to:
+
+**The Saints, Sages, Siddhas, Yogis, Sadhus, Rishis, Gurus, Mystics, and Spiritual Masters** of all genders, backgrounds, traditions, and walks of life whose timeless wisdom illuminates Anveshak. From ancient Sages to modern Masters, their selfless dedication to uplift humanity through selfless love and spiritual knowledge continues to guide seekers on the path.
+# ...
+""")
+```
+
+### Inclusive Recognition
+
+Anveshak explicitly acknowledges and honors spiritual teachers from all backgrounds:
+
+- All references to spiritual figures capitalize the first letter (Saints, Sages, etc.)
+- The application includes language acknowledging Masters of "all genders, backgrounds, traditions, and walks of life"
+- The selection of texts aims to represent diverse spiritual traditions
+
+From the Sources.py file:
+
+> "Additionally, there are and there have been many other great Saints, enlightened beings, Sadhus, Sages, and Gurus who have worked tirelessly to uplift humanity and guide beings to their true SELF and path, of whom little is known and documented. We thank them and acknowledge their contribution to the world."
+
+## Data Replication and Backup
+
+### GCS as Primary Storage
+
+Google Cloud Storage serves as both the primary storage and backup system:
+
+- All preprocessed data is stored in GCS buckets
+- GCS provides built-in redundancy and backup capabilities
+- Data is loaded from GCS at application startup
+
+### Local Caching
+
+For performance, Anveshak caches data locally using the implementation from rag_engine.py:
+
+```python
+def download_file_from_gcs(bucket, gcs_path, local_path):
+    """
+    Download a file from GCS to local storage if not already present.
+    
+    Only downloads if the file isn't already present locally, avoiding redundant downloads.
+    
+    Args:
+        bucket: GCS bucket object
+        gcs_path (str): Path to the file in GCS
+        local_path (str): Local path where the file should be saved
+        
+    Returns:
+        bool: True if download was successful or file already exists, False otherwise
+    """
+    try:
+        if os.path.exists(local_path):
+            print(f"File already exists locally: {local_path}")
+            return True
+            
+        blob = bucket.blob(gcs_path)
+        blob.download_to_filename(local_path)
+        print(f"✅ Downloaded {gcs_path} → {local_path}")
+        return True
+    except Exception as e:
+        print(f"❌ Error downloading {gcs_path}: {str(e)}")
+        return False
+```
+
+This approach:
+- Avoids redundant downloads
+- Preserves data across application restarts
+- Reduces API calls to GCS
+
+## Conclusion
+
+Anveshak: Spirituality Q&A implements a comprehensive data handling strategy that:
+
+1. **Respects Copyright**: Through word limits, citations, and acknowledgments
+2. **Preserves Source Integrity**: By maintaining accurate metadata and citations
+3. **Optimizes Performance**: Through efficient storage, retrieval, and caching
+4. **Ensures Ethical Use**: By focusing on educational purposes and proper attribution
+5. **Protects Privacy**: By not collecting or storing user data
+6. **Honors Diversity**: By acknowledging spiritual teachers of all backgrounds and traditions
+
+This balance between technical efficiency and ethical responsibility allows Anveshak to serve as a bridge to spiritual knowledge while respecting the original sources, traditions, and user privacy. The system is designed not to replace personal spiritual inquiry but to supplement it by making traditional wisdom more accessible.
+
+As stated in the conclusion of the blog post:
+
+> "The core philosophy guiding this project is that while technology can facilitate access to spiritual knowledge, the journey to self-discovery remains deeply personal. As Anveshak states: 'The path and journey to the SELF is designed to be undertaken alone. The all-encompassing knowledge is internal and not external.'"

scripts/preprocessing.ipynb

@@ -0,0 +1,644 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "L4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "\"\"\"\n",
+        "Anveshak: Spirituality Q&A - Data Preprocessing Pipeline\n",
+        "\n",
+        "This script processes the spiritual text corpus for the Anveshak application:\n",
+        "1. Uploads and downloads text files from various sources\n",
+        "2. Cleans and processes the texts to remove artifacts and noise\n",
+        "3. Chunks texts into smaller, manageable pieces\n",
+        "4. Generates embeddings using the E5-large-v2 model\n",
+        "5. Creates a FAISS index for efficient similarity search\n",
+        "6. Uploads all processed data to Google Cloud Storage\n",
+        "\n",
+        "Usage:\n",
+        "- Run in Google Colab with GPU runtime for faster embedding generation\n",
+        "- Ensure GCP authentication is set up before running\n",
+        "- Configure the constants below with your actual settings\n",
+        "\"\"\""
+      ],
+      "metadata": {
+        "id": "Cyjr-eDz9GmH"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =============================================================================\n",
+        "# CONFIGURATION SETTINGS\n",
+        "# =============================================================================\n",
+        "# Update these values with your actual settings\n",
+        "# Before open-sourcing, clear these values or replace with placeholders\n",
+        "BUCKET_NAME_GCS = \"your-bucket-name\"  # e.g., \"spiritual-texts-bucket\"\n",
+        "EMBEDDING_MODEL = \"your-embedding-model\"  # e.g., \"intfloat/e5-large-v2\"\n",
+        "# LLM_MODEL = \"your-llm-model\"  # e.g., \"gpt-3.5-turbo\"\n",
+        "\n",
+        "# GCS Paths - update these with your folder structure\n",
+        "METADATA_PATH_GCS = \"metadata/metadata.jsonl\"\n",
+        "RAW_TEXTS_UPLOADED_PATH_GCS = \"raw-texts/uploaded\"\n",
+        "RAW_TEXTS_DOWNLOADED_PATH_GCS = \"raw-texts/downloaded/\"\n",
+        "CLEANED_TEXTS_PATH_GCS = \"cleaned-texts/\"\n",
+        "EMBEDDINGS_PATH_GCS = \"processed/embeddings/all_embeddings.npy\"\n",
+        "INDICES_PATH_GCS = \"processed/indices/faiss_index.faiss\"\n",
+        "CHUNKS_PATH_GCS = \"processed/chunks/text_chunks.txt\"\n",
+        "\n",
+        "# Local file paths in Colab environment - update these with your folder structure\n",
+        "LOCAL_METADATA_FILE = \"/content/metadata.jsonl\"\n",
+        "LOCAL_RAW_TEXTS_FOLDER = \"/content/raw-texts/uploaded\"\n",
+        "LOCAL_EMBEDDINGS_FILE = \"/tmp/all_embeddings.npy\"\n",
+        "LOCAL_FAISS_INDEX_FILE = \"/tmp/faiss_index.faiss\"\n",
+        "LOCAL_TEXT_CHUNKS_FILE = \"/tmp/text_chunks.txt\""
+      ],
+      "metadata": {
+        "id": "YEDyIvmoXsPB"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H1tEbKhur8xf"
+      },
+      "outputs": [],
+      "source": [
+        "# Install required packages\n",
+        "!pip install faiss-cpu"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Import necessary libraries\n",
+        "from google.colab import files\n",
+        "from google.colab import auth\n",
+        "from google.cloud import storage\n",
+        "import os\n",
+        "import json\n",
+        "import requests\n",
+        "import re\n",
+        "import unicodedata\n",
+        "from bs4 import BeautifulSoup\n",
+        "import numpy as np\n",
+        "import faiss\n",
+        "import torch\n",
+        "from sentence_transformers import SentenceTransformer"
+      ],
+      "metadata": {
+        "id": "xCDTvZJRse4-"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =============================================================================\n",
+        "# AUTHENTICATION & INITIALIZATION\n",
+        "# =============================================================================\n",
+        "\n",
+        "# Authenticate with Google Cloud (only needed in Colab)\n",
+        "auth.authenticate_user()\n",
+        "\n",
+        "# Initialize GCS client (single initialization)\n",
+        "storage_client = storage.Client()\n",
+        "bucket = storage_client.bucket(BUCKET_NAME_GCS)"
+      ],
+      "metadata": {
+        "id": "hSYQ0ZSasjLd"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =============================================================================\n",
+        "# PART 1: UPLOAD RAW TEXTS AND METADATA\n",
+        "# =============================================================================\n",
+        "\n",
+        "def upload_files_to_colab():\n",
+        "    \"\"\"\n",
+        "    Upload raw text files and metadata from local machine to Colab.\n",
+        "\n",
+        "    This function:\n",
+        "    1. Prompts the user to upload text files\n",
+        "    2. Saves the uploaded files to a local directory\n",
+        "    3. Prompts the user to upload the metadata.jsonl file\n",
+        "    4. Saves the metadata file to the specified location\n",
+        "\n",
+        "    Returns:\n",
+        "        bool: True if upload was successful, False otherwise\n",
+        "    \"\"\"\n",
+        "    # First, upload text files\n",
+        "    print(\"Step 1: Please upload your text files...\")\n",
+        "    uploaded_text_files = files.upload()  # This will prompt the user to upload files\n",
+        "\n",
+        "    # Create directory structure if it doesn't exist\n",
+        "    os.makedirs(LOCAL_RAW_TEXTS_FOLDER, exist_ok=True)\n",
+        "\n",
+        "    # Move uploaded text files to the raw-texts folder\n",
+        "    for filename, content in uploaded_text_files.items():\n",
+        "        if filename.endswith(\".txt\"):\n",
+        "            with open(os.path.join(LOCAL_RAW_TEXTS_FOLDER, filename), \"wb\") as f:\n",
+        "                f.write(content)\n",
+        "            print(f\"✅ Saved {filename} to {LOCAL_RAW_TEXTS_FOLDER}\")\n",
+        "\n",
+        "    print(\"Text files upload complete!\")\n",
+        "\n",
+        "    # Next, upload metadata file\n",
+        "    print(\"\\nStep 2: Please upload your metadata.jsonl file...\")\n",
+        "    uploaded_metadata = files.upload()  # This will prompt the user to upload files\n",
+        "\n",
+        "    # Save metadata file\n",
+        "    metadata_uploaded = False\n",
+        "    for filename, content in uploaded_metadata.items():\n",
+        "        if filename == \"metadata.jsonl\":\n",
+        "            # Ensure the directory for metadata file exists\n",
+        "            os.makedirs(os.path.dirname(LOCAL_METADATA_FILE), exist_ok=True)\n",
+        "            with open(LOCAL_METADATA_FILE, \"wb\") as f:\n",
+        "                f.write(content)\n",
+        "            print(f\"✅ Saved metadata.jsonl to {LOCAL_METADATA_FILE}\")\n",
+        "            metadata_uploaded = True\n",
+        "\n",
+        "    if not metadata_uploaded:\n",
+        "        print(\"⚠️ Warning: metadata.jsonl was not uploaded. Please upload it to continue.\")\n",
+        "        return False\n",
+        "\n",
+        "    print(\"Upload to Colab complete!\")\n",
+        "    return True\n",
+        "\n",
+        "def upload_files_to_gcs():\n",
+        "    \"\"\"\n",
+        "    Upload raw text files and metadata from Colab to Google Cloud Storage.\n",
+        "\n",
+        "    This function:\n",
+        "    1. Uploads each text file from the local directory to GCS\n",
+        "    2. Uploads the metadata.jsonl file to GCS\n",
+        "\n",
+        "    All files are uploaded to the paths specified in the configuration constants.\n",
+        "    \"\"\"\n",
+        "    # Upload each file from the local raw-texts folder to GCS\n",
+        "    for filename in os.listdir(LOCAL_RAW_TEXTS_FOLDER):\n",
+        "        local_path = os.path.join(LOCAL_RAW_TEXTS_FOLDER, filename)\n",
+        "        blob_path = f\"{RAW_TEXTS_UPLOADED_PATH_GCS}/{filename}\"  # GCS path\n",
+        "        blob = bucket.blob(blob_path)\n",
+        "        try:\n",
+        "            blob.upload_from_filename(local_path)\n",
+        "            print(f\"✅ Uploaded: {filename} -> gs://{BUCKET_NAME_GCS}/{blob_path}\")\n",
+        "        except Exception as e:\n",
+        "            print(f\"❌ Failed to upload {filename}: {e}\")\n",
+        "\n",
+        "    # Upload metadata file\n",
+        "    blob = bucket.blob(METADATA_PATH_GCS)\n",
+        "    try:\n",
+        "        blob.upload_from_filename(LOCAL_METADATA_FILE)\n",
+        "        print(f\"✅ Uploaded metadata.jsonl -> gs://{BUCKET_NAME_GCS}/{METADATA_PATH_GCS}\")\n",
+        "    except Exception as e:\n",
+        "        print(f\"❌ Failed to upload metadata: {e}\")"
+      ],
+      "metadata": {
+        "id": "cShc029islmO"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =============================================================================\n",
+        "# PART 2: DOWNLOAD AND CLEAN TEXTS\n",
+        "# =============================================================================\n",
+        "\n",
+        "def fetch_metadata_from_gcs():\n",
+        "    \"\"\"\n",
+        "    Fetch metadata.jsonl from GCS and return as a list of dictionaries.\n",
+        "\n",
+        "    Each dictionary represents a text entry with metadata like title, author, etc.\n",
+        "\n",
+        "    Returns:\n",
+        "        list: List of dictionaries containing metadata for each text\n",
+        "    \"\"\"\n",
+        "    blob = bucket.blob(METADATA_PATH_GCS)\n",
+        "    # Download metadata file\n",
+        "    metadata_jsonl = blob.download_as_text()\n",
+        "    # Parse JSONL\n",
+        "    metadata = [json.loads(line) for line in metadata_jsonl.splitlines()]\n",
+        "    return metadata\n",
+        "\n",
+        "def upload_to_gcs(source_file, destination_path):\n",
+        "    \"\"\"\n",
+        "    Upload a local file to Google Cloud Storage.\n",
+        "\n",
+        "    Args:\n",
+        "        source_file (str): Path to the local file\n",
+        "        destination_path (str): Path in GCS where the file should be uploaded\n",
+        "    \"\"\"\n",
+        "    blob = bucket.blob(destination_path)\n",
+        "    blob.upload_from_filename(source_file)\n",
+        "    print(f\"📤 Uploaded to GCS: {destination_path}\")\n",
+        "\n",
+        "def download_text_files():\n",
+        "    \"\"\"\n",
+        "    Download text files from URLs specified in the metadata.\n",
+        "\n",
+        "    This function:\n",
+        "    1. Fetches metadata from GCS\n",
+        "    2. Filters entries where Uploaded=False (texts to be downloaded)\n",
+        "    3. Downloads each text from its URL\n",
+        "    4. Uploads the downloaded text to GCS\n",
+        "\n",
+        "    This allows automated collection of texts that weren't manually uploaded.\n",
+        "    \"\"\"\n",
+        "    metadata = fetch_metadata_from_gcs()\n",
+        "    # Filter entries where Uploaded is False\n",
+        "    files_to_download = [item for item in metadata if item[\"Uploaded\"] == False]\n",
+        "    print(f\"🔍 Found {len(files_to_download)} files to download\")\n",
+        "\n",
+        "    # Process only necessary files\n",
+        "    for item in files_to_download:\n",
+        "        name, author, url = item[\"Title\"], item[\"Author\"], item[\"URL\"]\n",
+        "        if url.lower() == \"not available\":\n",
+        "            print(f\"❌ Skipping {name} - No URL available.\")\n",
+        "            continue\n",
+        "\n",
+        "        try:\n",
+        "            response = requests.get(url)\n",
+        "            if response.status_code == 200:\n",
+        "                raw_text = response.text\n",
+        "                filename = \"{}.txt\".format(name.replace(\" \", \"_\"))\n",
+        "                # Save to local first\n",
+        "                local_path = f\"/tmp/{filename}\"\n",
+        "                with open(local_path, \"w\", encoding=\"utf-8\") as file:\n",
+        "                    file.write(raw_text)\n",
+        "                # Upload to GCS\n",
+        "                gcs_path = f\"{RAW_TEXTS_DOWNLOADED_PATH_GCS}{filename}\"\n",
+        "                upload_to_gcs(local_path, gcs_path)\n",
+        "                print(f\"✅ Downloaded & uploaded: {filename} ({len(raw_text.split())} words)\")\n",
+        "                # Clean up temp file\n",
+        "                os.remove(local_path)\n",
+        "            else:\n",
+        "                print(f\"❌ Failed to download {name}: {url} (Status {response.status_code})\")\n",
+        "        except Exception as e:\n",
+        "            print(f\"❌ Error processing {name}: {e}\")\n",
+        "\n",
+        "def rigorous_clean_text(text):\n",
+        "    \"\"\"\n",
+        "    Clean text by removing metadata, junk text, and formatting issues.\n",
+        "\n",
+        "    This function:\n",
+        "    1. Removes HTML tags using BeautifulSoup\n",
+        "    2. Removes URLs and standalone numbers\n",
+        "    3. Removes all-caps OCR noise words\n",
+        "    4. Deduplicates adjacent identical lines\n",
+        "    5. Normalizes Unicode characters\n",
+        "    6. Standardizes whitespace and newlines\n",
+        "\n",
+        "    Args:\n",
+        "        text (str): The raw text to clean\n",
+        "\n",
+        "    Returns:\n",
+        "        str: The cleaned text\n",
+        "    \"\"\"\n",
+        "    text = BeautifulSoup(text, \"html.parser\").get_text()\n",
+        "    text = re.sub(r\"https?:\\/\\/\\S+\", \"\", text)  # Remove links\n",
+        "    text = re.sub(r\"\\b\\d+\\b\", \"\", text)  # Remove standalone numbers\n",
+        "    text = re.sub(r\"\\b[A-Z]{5,}\\b\", \"\", text)  # Remove all-caps OCR noise words\n",
+        "    lines = text.split(\"\\n\")\n",
+        "    cleaned_lines = []\n",
+        "    last_line = None\n",
+        "\n",
+        "    for line in lines:\n",
+        "        line = line.strip()\n",
+        "        if line and line != last_line:\n",
+        "            cleaned_lines.append(line)\n",
+        "            last_line = line\n",
+        "\n",
+        "    text = \"\\n\".join(cleaned_lines)\n",
+        "    text = unicodedata.normalize(\"NFKD\", text)\n",
+        "    text = re.sub(r\"\\s+\", \" \", text).strip()\n",
+        "    text = re.sub(r\"\\n{2,}\", \"\\n\", text)\n",
+        "    return text\n",
+        "\n",
+        "def clean_and_upload_texts():\n",
+        "    \"\"\"\n",
+        "    Download raw texts from GCS, clean them, and upload cleaned versions back to GCS.\n",
+        "\n",
+        "    This function processes all texts in both the uploaded and downloaded folders:\n",
+        "    1. For each text file, downloads it from GCS\n",
+        "    2. Cleans the text using rigorous_clean_text()\n",
+        "    3. Uploads the cleaned version back to GCS in the cleaned-texts folder\n",
+        "\n",
+        "    This step ensures that all texts are properly formatted before embedding generation.\n",
+        "    \"\"\"\n",
+        "    raw_texts_folders = [RAW_TEXTS_DOWNLOADED_PATH_GCS, RAW_TEXTS_UPLOADED_PATH_GCS]  # Process both folders\n",
+        "    total_files = 0  # Counter to track number of processed files\n",
+        "\n",
+        "    for raw_texts_folder in raw_texts_folders:\n",
+        "        # List all files in the current raw-texts folder\n",
+        "        blobs = list(bucket.list_blobs(prefix=raw_texts_folder))\n",
+        "        print(f\"🔍 Found {len(blobs)} files in {raw_texts_folder}\")\n",
+        "\n",
+        "        for blob in blobs:\n",
+        "            if not blob.name.endswith(\".txt\"):  # Skip non-text files\n",
+        "                continue\n",
+        "\n",
+        "            try:\n",
+        "                # Download file\n",
+        "                raw_text = blob.download_as_text().strip()\n",
+        "                if not raw_text:  # Skip empty files\n",
+        "                    print(f\"⚠️ Skipping empty file: {blob.name}\")\n",
+        "                    continue\n",
+        "\n",
+        "                # Clean text\n",
+        "                cleaned_text = rigorous_clean_text(raw_text)\n",
+        "\n",
+        "                # Save cleaned text back to GCS\n",
+        "                cleaned_blob_name = blob.name.replace(raw_texts_folder, CLEANED_TEXTS_PATH_GCS)\n",
+        "                cleaned_blob = bucket.blob(cleaned_blob_name)\n",
+        "                cleaned_blob.upload_from_string(cleaned_text, content_type=\"text/plain\")\n",
+        "                print(f\"✅ Cleaned & uploaded: {cleaned_blob_name} ({len(cleaned_text.split())} words, {len(cleaned_text)} characters)\")\n",
+        "                total_files += 1\n",
+        "            except Exception as e:\n",
+        "                print(f\"❌ Error processing {blob.name}: {e}\")\n",
+        "\n",
+        "    print(f\"🚀 Cleaning process completed! Total cleaned & uploaded files: {total_files}\")"
+      ],
+      "metadata": {
+        "id": "Vskwg984s25K"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =============================================================================\n",
+        "# PART 3: GENERATE EMBEDDINGS AND INDEX\n",
+        "# =============================================================================\n",
+        "\n",
+        "def fetch_metadata_dict_from_gcs():\n",
+        "    \"\"\"\n",
+        "    Fetch metadata.jsonl from GCS and return as a dictionary.\n",
+        "\n",
+        "    The dictionary is keyed by title for easy lookup during text processing.\n",
+        "\n",
+        "    Returns:\n",
+        "        dict: Dictionary mapping text titles to their metadata\n",
+        "    \"\"\"\n",
+        "    metadata_blob = bucket.blob(METADATA_PATH_GCS)\n",
+        "    metadata_dict = {}\n",
+        "\n",
+        "    if metadata_blob.exists():\n",
+        "        metadata_content = metadata_blob.download_as_text()\n",
+        "        for line in metadata_content.splitlines():\n",
+        "            item = json.loads(line)\n",
+        "            metadata_dict[item[\"Title\"]] = item  # Keep space-based lookup\n",
+        "    else:\n",
+        "        print(\"❌ Metadata file not found in GCS\")\n",
+        "\n",
+        "    return metadata_dict\n",
+        "\n",
+        "def chunk_text(text, chunk_size=500, overlap=50):\n",
+        "    \"\"\"\n",
+        "    Split text into smaller, overlapping chunks for better retrieval.\n",
+        "\n",
+        "    Args:\n",
+        "        text (str): The text to chunk\n",
+        "        chunk_size (int): Maximum number of words per chunk\n",
+        "        overlap (int): Number of words to overlap between chunks\n",
+        "\n",
+        "    Returns:\n",
+        "        list: List of text chunks\n",
+        "    \"\"\"\n",
+        "    words = text.split()\n",
+        "    chunks = []\n",
+        "    i = 0\n",
+        "\n",
+        "    while i < len(words):\n",
+        "        chunk = \" \".join(words[i:i + chunk_size])\n",
+        "        chunks.append(chunk)\n",
+        "        i += chunk_size - overlap\n",
+        "\n",
+        "    return chunks\n",
+        "\n",
+        "def create_embeddings(text_chunks, batch_size=32):\n",
+        "    \"\"\"\n",
+        "    Generate embeddings for the given chunks of text using the specified embedding model.\n",
+        "\n",
+        "    This function:\n",
+        "    1. Uses SentenceTransformer to load the embedding model\n",
+        "    2. Prefixes each chunk with \"passage:\" as required by the E5 model\n",
+        "    3. Processes chunks in batches to manage memory usage\n",
+        "    4. Normalizes embeddings for cosine similarity search\n",
+        "\n",
+        "    Args:\n",
+        "        text_chunks (list): List of text chunks to embed\n",
+        "        batch_size (int): Number of chunks to process at once\n",
+        "\n",
+        "    Returns:\n",
+        "        numpy.ndarray: Matrix of embeddings, one per text chunk\n",
+        "    \"\"\"\n",
+        "    # Load the model with GPU optimization\n",
+        "    model = SentenceTransformer(EMBEDDING_MODEL)\n",
+        "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+        "    model = model.to(device)\n",
+        "    print(f\"🚀 Using device for embeddings: {device}\")\n",
+        "\n",
+        "    prefixed_chunks = [f\"passage: {text}\" for text in text_chunks]\n",
+        "    all_embeddings = []\n",
+        "\n",
+        "    for i in range(0, len(prefixed_chunks), batch_size):\n",
+        "        batch = prefixed_chunks[i:i+batch_size]\n",
+        "\n",
+        "        # Move batch to GPU (if available) for faster processing\n",
+        "        with torch.no_grad():\n",
+        "            batch_embeddings = model.encode(batch, convert_to_numpy=True, normalize_embeddings=True)\n",
+        "\n",
+        "        all_embeddings.append(batch_embeddings)\n",
+        "\n",
+        "        if (i + batch_size) % 100 == 0 or (i + batch_size) >= len(prefixed_chunks):\n",
+        "            print(f\"📌 Processed {i + min(batch_size, len(prefixed_chunks) - i)}/{len(prefixed_chunks)} documents\")\n",
+        "\n",
+        "    return np.vstack(all_embeddings).astype(\"float32\")\n",
+        "\n",
+        "def process_cleaned_texts():\n",
+        "    \"\"\"\n",
+        "    Process cleaned texts to create embeddings, FAISS index, and text chunks with metadata.\n",
+        "\n",
+        "    This function:\n",
+        "    1. Downloads all cleaned texts from GCS\n",
+        "    2. Chunks each text into smaller pieces\n",
+        "    3. Generates embeddings for each chunk\n",
+        "    4. Creates a FAISS index for similarity search\n",
+        "    5. Saves and uploads all processed data back to GCS\n",
+        "\n",
+        "    This is the core processing step that prepares data for the RAG system.\n",
+        "    \"\"\"\n",
+        "    all_chunks = []\n",
+        "    all_metadata = []\n",
+        "    chunk_counter = 0\n",
+        "\n",
+        "    metadata_dict = fetch_metadata_dict_from_gcs()  # Load metadata\n",
+        "\n",
+        "    # Optimized listing of blobs in cleaned-texts folder\n",
+        "    blobs = list(storage_client.list_blobs(BUCKET_NAME_GCS, prefix=CLEANED_TEXTS_PATH_GCS))\n",
+        "    print(f\"🔍 Found {len(blobs)} files in {CLEANED_TEXTS_PATH_GCS}\")\n",
+        "\n",
+        "    if not blobs:\n",
+        "        print(f\"❌ No files found in {CLEANED_TEXTS_PATH_GCS}. Exiting.\")\n",
+        "        return\n",
+        "\n",
+        "    for blob in blobs:\n",
+        "        file_name = blob.name.split(\"/\")[-1]\n",
+        "        if not file_name or file_name.startswith(\".\"):\n",
+        "            continue  # Skip empty or hidden files\n",
+        "\n",
+        "        # Convert filename back to space-based title for metadata lookup\n",
+        "        book_name = file_name.replace(\"_\", \" \")\n",
+        "        metadata = metadata_dict.get(book_name, {\"Author\": \"Unknown\", \"Publisher\": \"Unknown\"})\n",
+        "        author = metadata.get(\"Author\", \"Unknown\")\n",
+        "\n",
+        "        try:\n",
+        "            # Download and read text\n",
+        "            raw_text = blob.download_as_text().strip()\n",
+        "\n",
+        "            # Skip empty or corrupt files\n",
+        "            if not raw_text:\n",
+        "                print(f\"❌ Skipping empty file: {file_name}\")\n",
+        "                continue\n",
+        "\n",
+        "            chunks = chunk_text(raw_text)\n",
+        "            print(f\"✅ Processed {book_name}: {len(chunks)} chunks\")\n",
+        "\n",
+        "            for chunk in chunks:\n",
+        "                all_chunks.append(chunk)\n",
+        "                all_metadata.append((chunk_counter, book_name, author))\n",
+        "                chunk_counter += 1\n",
+        "        except Exception as e:\n",
+        "            print(f\"❌ Error processing {file_name}: {e}\")\n",
+        "\n",
+        "    # Ensure there are chunks before embedding generation\n",
+        "    if not all_chunks:\n",
+        "        print(\"❌ No chunks found. Skipping embedding generation.\")\n",
+        "        return\n",
+        "\n",
+        "    # Create embeddings with GPU acceleration\n",
+        "    print(f\"📍 Creating embeddings for {len(all_chunks)} total chunks...\")\n",
+        "    all_embeddings = create_embeddings(all_chunks)\n",
+        "\n",
+        "    # Build FAISS index\n",
+        "    dimension = all_embeddings.shape[1]\n",
+        "    index = faiss.IndexFlatIP(dimension)\n",
+        "    index.add(all_embeddings)\n",
+        "    print(f\"✅ FAISS index built with {index.ntotal} vectors\")\n",
+        "\n",
+        "    # Save & upload embeddings\n",
+        "    np.save(LOCAL_EMBEDDINGS_FILE, all_embeddings)  # Save locally first\n",
+        "    embeddings_blob = bucket.blob(EMBEDDINGS_PATH_GCS)\n",
+        "    embeddings_blob.upload_from_filename(LOCAL_EMBEDDINGS_FILE)\n",
+        "    print(f\"✅ Uploaded embeddings to GCS: {EMBEDDINGS_PATH_GCS}\")\n",
+        "\n",
+        "    # Save & upload FAISS index\n",
+        "    faiss.write_index(index, LOCAL_FAISS_INDEX_FILE)\n",
+        "    index_blob = bucket.blob(INDICES_PATH_GCS)\n",
+        "    index_blob.upload_from_filename(LOCAL_FAISS_INDEX_FILE)\n",
+        "    print(f\"✅ Uploaded FAISS index to GCS: {INDICES_PATH_GCS}\")\n",
+        "\n",
+        "    # Save and upload text chunks with metadata\n",
+        "    with open(LOCAL_TEXT_CHUNKS_FILE, \"w\", encoding=\"utf-8\") as f:\n",
+        "        for i, (chunk_id, book_name, author) in enumerate(all_metadata):\n",
+        "            f.write(f\"{i}\\t{book_name}\\t{author}\\t{all_chunks[i]}\\n\")\n",
+        "\n",
+        "    chunks_blob = bucket.blob(CHUNKS_PATH_GCS)\n",
+        "    chunks_blob.upload_from_filename(LOCAL_TEXT_CHUNKS_FILE)\n",
+        "    print(f\"✅ Uploaded text chunks to GCS: {CHUNKS_PATH_GCS}\")\n",
+        "\n",
+        "    # Clean up temp files\n",
+        "    os.remove(LOCAL_EMBEDDINGS_FILE)\n",
+        "    os.remove(LOCAL_FAISS_INDEX_FILE)\n",
+        "    os.remove(LOCAL_TEXT_CHUNKS_FILE)"
+      ],
+      "metadata": {
+        "id": "1Yul8p9JsN1e"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =============================================================================\n",
+        "# PART 4: MAIN EXECUTION\n",
+        "# =============================================================================\n",
+        "\n",
+        "def run_pipeline():\n",
+        "    \"\"\"\n",
+        "    Run the complete end-to-end preprocessing pipeline.\n",
+        "\n",
+        "    This function executes all steps in sequence:\n",
+        "    1. Upload files from local to Colab\n",
+        "    2. Upload raw texts and metadata to GCS\n",
+        "    3. Download texts from URLs specified in metadata\n",
+        "    4. Clean and process all texts\n",
+        "    5. Generate embeddings and build the FAISS index\n",
+        "\n",
+        "    This is the main entry point for the preprocessing script.\n",
+        "    \"\"\"\n",
+        "    print(\"🚀 Starting pipeline execution...\")\n",
+        "\n",
+        "    print(\"\\n==== STEP 1: Uploading files from local to Colab ====\")\n",
+        "    upload_successful = upload_files_to_colab()\n",
+        "\n",
+        "    if not upload_successful:\n",
+        "        print(\"❌ Pipeline halted due to missing metadata file.\")\n",
+        "        return\n",
+        "\n",
+        "    print(\"\\n==== STEP 2: Uploading raw texts and metadata to GCS ====\")\n",
+        "    upload_files_to_gcs()\n",
+        "\n",
+        "    print(\"\\n==== STEP 3: Downloading texts from URLs ====\")\n",
+        "    download_text_files()\n",
+        "\n",
+        "    print(\"\\n==== STEP 4: Cleaning and processing texts ====\")\n",
+        "    clean_and_upload_texts()\n",
+        "\n",
+        "    print(\"\\n==== STEP 5: Generating embeddings and building index ====\")\n",
+        "    process_cleaned_texts()\n",
+        "\n",
+        "    print(\"\\n✅ Pipeline execution completed successfully!\")\n",
+        "\n",
+        "# Execute the complete pipeline\n",
+        "if __name__ == \"__main__\":\n",
+        "    run_pipeline()"
+      ],
+      "metadata": {
+        "id": "XXB_eYvj-I0i"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}