Upload 27 files

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.12.9-slim-bookworm
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN mkdir -p /app/logs && chmod 777 /app/logs
+COPY . .
+EXPOSE 8080
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,51 @@
+from flask import Flask, render_template, request
+from knowledgeassistant.components.summarization import DataSummarization
+from knowledgeassistant.components.keywordextraction import KeywordExtraction
+from knowledgeassistant.components.RAG import RAG
+from knowledgeassistant.entity.config_entity import DataSummarizationConfig, PipelineConfig, DataInputConfig, KeywordExtractionConfig, RAGConfig
+from knowledgeassistant.utils.main_utils.utils import write_txt_file, read_txt_file
+
+app = Flask(__name__)
+
+@app.route('/', methods=['GET', 'POST'])
+def home():
+    if request.method == 'POST':
+        pipeline_config = PipelineConfig()
+        data_input_config = DataInputConfig(pipeline_config)
+
+        user_text = request.form['user_text']
+        user_option = request.form['tasks']
+        result_text = ""
+
+        if user_option == "Q&A":
+            user_question = request.form['user_question']
+        else:
+            user_number = int(request.form['user_number'])
+
+        input_text_path = data_input_config.input_text_file_path
+        write_txt_file(input_text_path, user_text, replace=True)
+
+        if user_option == "summarization":
+            data_summarization_config = DataSummarizationConfig(pipeline_config)
+            summarizer = DataSummarization(data_summarization_config)
+            file_path = data_summarization_config.summarized_text_file_path
+            summarizer.initiate_data_summarization(input_text_path=input_text_path, min_length=int(user_number))
+        elif user_option == "keywords":
+            keyword_extraction_config = KeywordExtractionConfig(pipeline_config)
+            extractor = KeywordExtraction(keyword_extraction_config)
+            file_path = keyword_extraction_config.extracted_keywords_file_path
+            extractor.initiate_keyword_extraction(input_text_path=input_text_path, keywords_count=int(user_number))
+        elif user_option == "Q&A":
+            rag_config = RAGConfig(pipeline_config)
+            rag = RAG(rag_config)
+            file_path = rag_config.rag_generated_text_path
+            rag.initiate_rag(input_text_path=input_text_path, query=user_question)
+
+        result_text = read_txt_file(file_path=file_path)
+
+        return render_template("index.html", task=user_option, text=result_text, user_text=user_text)
+
+    return render_template("index.html", task=None, text=None, user_text=None)
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=8080)

knowledgeassistant/components/RAG.py

@@ -0,0 +1,92 @@
+from knowledgeassistant.logging.logger import logging
+from knowledgeassistant.exception.exception import KnowledgeAssistantException
+
+from knowledgeassistant.entity.config_entity import RAGConfig
+from knowledgeassistant.utils.main_utils.utils import read_txt_file, write_txt_file
+
+import os
+import sys
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+from langchain_ollama import OllamaEmbeddings
+from langchain_community.vectorstores import FAISS
+from together import Together
+from langchain.chains import RetrievalQA
+from langchain_core.language_models import LLM
+
+from dotenv import load_dotenv
+import typing
+
+load_dotenv()
+os.environ["TOGETHER_API_KEY"] = os.getenv("TOGETHER_API_KEY")
+
+class RAG:
+    def __init__(self, rag_config: RAGConfig):
+        try:
+            self.rag_config = rag_config
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+        
+    def split_text(self, input_text_path: str):
+        try:
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size = 1000, chunk_overlap = 200)
+            raw_documents = text_splitter.split_text(text = read_txt_file(file_path = input_text_path))
+            documents = [Document(page_content=text) for text in raw_documents]
+            return documents
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+    
+    def create_and_store_embeddings(self, documents: list):
+        try:
+            db = FAISS.from_documents(documents, OllamaEmbeddings(model="nomic-embed-text"))
+            return db
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+
+    class TogetherLLM(LLM):
+        model_name: str = "meta-llama/Llama-3-8b-chat-hf"
+
+        @property
+        def _llm_type(self) -> str:
+            return "together_ai"
+
+        def _call(self, prompt: str, stop: typing.Optional[typing.List[str]] = None) -> str:
+            client = Together()
+            response = client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return response.choices[0].message.content
+    
+    def retrieval(self, llm, db, query):
+        try:
+            chain = RetrievalQA.from_chain_type(
+                llm=llm,
+                retriever=db.as_retriever()
+            )
+            result = chain.invoke(query)
+            return result
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+    
+    def initiate_rag(self, input_text_path: str, query: str):
+        try:
+            docs = self.split_text(input_text_path = input_text_path)
+            logging.info("Splitted Text into Chunks Successfully")
+            store = self.create_and_store_embeddings(documents = docs)
+            logging.info("Successfully stored vector embeddings")
+            llm = self.TogetherLLM()
+            logging.info("Successfully loaded the llm")
+            result = self.retrieval(
+                llm = llm,
+                db = store,
+                query = query
+            )
+            logging.info("Successfully Generated Results")
+            write_txt_file(
+                file_path = self.rag_config.rag_generated_text_path,
+                content = result['result']
+            )
+            logging.info("Successfully wrote results in txt file")
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)

knowledgeassistant/components/keywordextraction.py

@@ -0,0 +1,51 @@
+from knowledgeassistant.exception.exception import KnowledgeAssistantException
+from knowledgeassistant.logging.logger import logging
+
+from knowledgeassistant.entity.config_entity import KeywordExtractionConfig
+from knowledgeassistant.utils.main_utils.utils import read_txt_file, write_txt_file
+
+import sys
+import numpy as np
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+
+class KeywordExtraction:
+    def __init__(self, keyword_extraction_config: KeywordExtractionConfig):
+        try:
+            self.keyword_extraction_config = keyword_extraction_config
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+    
+    def extract_keywords(self, input_text_path: str, keywords_count: str):
+        try:
+            vectorizer = CountVectorizer(stop_words='english', token_pattern=r'(?u)\b[a-zA-Z]+\b')
+            transformer = TfidfTransformer()
+            logging.info("Vectorizer and Transformer successfully setup")
+
+            text = read_txt_file(file_path=input_text_path)
+            word_count_matrix = vectorizer.fit_transform([text])
+            tfidf_matrix = transformer.fit_transform(word_count_matrix)
+            logging.info("Successfully calculated word count and their tfidf scores")
+
+            feature_array = np.array([word for word in vectorizer.get_feature_names_out() if word.isalpha()])
+            tfidf_sorting = np.argsort(tfidf_matrix.toarray()).flatten()[::-1]
+            logging.info("Successfully extracted keywords and sorted in descending order of their tfidf scores")
+
+            top_keywords = feature_array[tfidf_sorting][:keywords_count]
+            content = "\n".join(top_keywords)
+            write_txt_file(
+                self.keyword_extraction_config.extracted_keywords_file_path,
+                content,
+                True
+            )
+            logging.info(f"Successfully extracted and wrote top {keywords_count} keywords from text")
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+    
+    def initiate_keyword_extraction(self, input_text_path: str, keywords_count: str):
+        try:
+            self.extract_keywords(
+                input_text_path = input_text_path,
+                keywords_count = keywords_count
+            )
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)

knowledgeassistant/components/summarization.py

@@ -0,0 +1,38 @@
+from knowledgeassistant.exception.exception import KnowledgeAssistantException
+from knowledgeassistant.logging.logger import logging
+
+from knowledgeassistant.entity.config_entity import DataSummarizationConfig
+from knowledgeassistant.utils.main_utils.utils import write_txt_file, read_txt_file
+
+import sys
+from transformers import pipeline
+
+class DataSummarization:
+    def __init__(self, data_summarization_config: DataSummarizationConfig):
+        try:
+            self.data_summarization_config = data_summarization_config
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+
+    def summarize(self, input_text_path: str, min_length: int):
+        try:
+            pipe = pipeline("summarization", model="facebook/bart-large-cnn")
+            logging.info("Summarization Pipeline Successfully Setup")
+
+            text = read_txt_file(input_text_path)
+            summary = pipe(text, min_length = min_length, do_sample = False)
+            logging.info("Text successfully summarized")
+            
+            write_txt_file(self.data_summarization_config.summarized_text_file_path, summary[0].get("summary_text"))
+            logging.info("Successfully wrote summarized text")
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)
+    
+    def initiate_data_summarization(self, input_text_path: str, min_length: int):
+        try:
+            self.summarize(
+                input_text_path = input_text_path,
+                min_length = min_length
+            )
+        except Exception as e:
+            raise KnowledgeAssistantException(e, sys)

knowledgeassistant/constant/pipeline/__init__.py

@@ -0,0 +1,19 @@
+PIPELINE_NAME: str = "KnowledgeAssistant"
+ARTIFACT_DIR: str = "Artifacts"
+
+
+INPUT_TEXT_DIR: str = "input_text"
+INPUT_TEXT_FILE_NAME: str = "input_text.txt"
+
+
+DATA_SUMMARIZATION_DIR_NAME: str = "data_summarization"
+SUMMARIZED_TEXT_DIR: str = "summarized_text"
+SUMMARIZED_TEXT_FILE_NAME: str = "summarized_text.txt"
+
+KEYWORD_EXTRACTION_DIR_NAME: str = "keyword_extraction"
+EXTRACTED_KEYWORDS_DIR: str = "keywords"
+EXTRACTED_KEYWORDS_FILE_NAME: str = "keywords.txt"
+
+RAG_DIR_NAME: str = "RAG"
+RAG_DIR: str = "generation"
+RAG_FILE_NAME: str = "generated_text.txt"

knowledgeassistant/entity/config_entity.py

@@ -0,0 +1,58 @@
+import os
+from datetime import datetime
+from knowledgeassistant.constant import pipeline
+
+class PipelineConfig:
+    def __init__(self, timestamp = datetime.now()):
+        timestamp = timestamp.strftime("%d_%m_%Y_%H_%M_%S")
+        self.pipeline_name = pipeline.PIPELINE_NAME
+        self.artifact_name = pipeline.ARTIFACT_DIR
+        self.artifact_dir = os.path.join(
+            self.artifact_name,
+            timestamp
+        )
+        self.timestamp: str = timestamp
+
+class DataInputConfig:
+    def __init__(self, pipeline_config: PipelineConfig):
+        self.input_text_file_path: str = os.path.join(
+            pipeline_config.artifact_dir,
+            pipeline.INPUT_TEXT_DIR,
+            pipeline.INPUT_TEXT_FILE_NAME
+        )
+
+class DataSummarizationConfig:
+    def __init__(self, pipeline_config: PipelineConfig):
+        self.data_summarization_dir: str = os.path.join(
+            pipeline_config.artifact_dir,
+            pipeline.DATA_SUMMARIZATION_DIR_NAME
+        )
+        self.summarized_text_file_path: str = os.path.join(
+            self.data_summarization_dir,
+            pipeline.SUMMARIZED_TEXT_DIR,
+            pipeline.SUMMARIZED_TEXT_FILE_NAME
+        )
+
+class KeywordExtractionConfig:
+    def __init__(self, pipeline_config: PipelineConfig):
+        self.keyword_extraction_dir: str = os.path.join(
+            pipeline_config.artifact_dir,
+            pipeline.KEYWORD_EXTRACTION_DIR_NAME
+        )
+        self.extracted_keywords_file_path: str = os.path.join(
+            self.keyword_extraction_dir,
+            pipeline.EXTRACTED_KEYWORDS_DIR,
+            pipeline.EXTRACTED_KEYWORDS_FILE_NAME
+        )
+
+class RAGConfig:
+    def __init__(self, pipeline_config: PipelineConfig):
+        self.RAG_dir: str = os.path.join(
+            pipeline_config.artifact_dir,
+            pipeline.RAG_DIR_NAME
+        )
+        self.rag_generated_text_path: str = os.path.join(
+            self.RAG_dir,
+            pipeline.RAG_DIR,
+            pipeline.RAG_FILE_NAME
+        )

knowledgeassistant/exception/exception.py

@@ -0,0 +1,15 @@
+import sys
+from knowledgeassistant.logging import logger
+
+class KnowledgeAssistantException(Exception):
+    def __init__(self,error_message,error_details:sys):
+        self.error_message = error_message
+        _,_,exc_tb = error_details.exc_info()
+        
+        self.lineno=exc_tb.tb_lineno
+        self.file_name=exc_tb.tb_frame.f_code.co_filename 
+    
+    def __str__(self):
+        return "Error occured in python script name [{0}] line number [{1}] error message [{2}]".format(
+            self.file_name, self.lineno, str(self.error_message)
+        )

knowledgeassistant/logging/logger.py

@@ -0,0 +1,16 @@
+import os
+import logging
+from datetime import datetime
+
+LOG_FILE = f"{datetime.now().strftime('%d_%m_%Y_%H_%M_%S')}.log"
+
+logs_path = os.path.join(os.getcwd(),"logs",LOG_FILE)
+os.makedirs(logs_path, exist_ok=True)
+
+LOG_FILE_PATH = os.path.join(logs_path, LOG_FILE)
+
+logging.basicConfig(
+    filename = LOG_FILE_PATH,
+    format = "[ %(asctime)s ] : %(lineno)d : %(name)s : %(levelname)s : %(module)s : %(message)s",
+    level = logging.INFO,
+)

knowledgeassistant/utils/main_utils/utils.py

@@ -0,0 +1,26 @@
+from knowledgeassistant.exception.exception import KnowledgeAssistantException
+from knowledgeassistant.logging.logger import logging
+import os
+import sys
+
+def write_txt_file(file_path: str, content: str, replace: bool = False):
+    try:
+        if replace and os.path.exists(file_path):
+            os.remove(file_path)
+            logging.info(f"Successfully removed the file: {file_path}")
+        dir_path = os.path.dirname(file_path)
+        logging.info(f"Successfully create the file: {file_path}")
+        os.makedirs(dir_path, exist_ok = True)
+        with open(file_path, 'w', encoding='utf-8') as txt_file:
+            txt_file.write(content)
+    except Exception as e:
+        raise KnowledgeAssistantException(e, sys)
+
+def read_txt_file(file_path: str) -> str:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as txt_file:
+            text = txt_file.read()
+            logging.info("Successfully read the file")
+            return text
+    except Exception as e:
+        raise KnowledgeAssistantException(e, sys)

requirements.txt

@@ -0,0 +1,18 @@
+ipykernel
+pandas
+numpy
+flask
+python-dotenv
+transformers
+torch
+torchvision
+torchaudio
+--extra-index-url https://download.pytorch.org/whl/cu126
+scikit-learn
+langchain
+langchain-community
+langchain-core
+ollama
+chromadb
+faiss-cpu
+together

static/script.js

@@ -0,0 +1,43 @@
+document.addEventListener("DOMContentLoaded", function () {
+    function toggleElements() {
+        let taskDropdown = document.querySelector("select[name='tasks']");
+        let numberInput = document.querySelector("input[name='user_number']");
+        let numberLabel = document.getElementById("number_label");
+        let questionInput = document.querySelector("input[name='user_question']");
+        let questionLabel = document.getElementById("question_label");
+        let submitButton = document.getElementById("submit_btn");
+
+        if (taskDropdown.value === "summarization") {
+            numberInput.style.display = "inline";
+            numberLabel.style.display = "inline";
+            numberInput.disabled = false;
+            numberLabel.textContent = "Enter minimum length: ";
+            questionInput.style.display = "none";
+            questionLabel.style.display = "none";
+            submitButton.disabled = false;
+        } else if (taskDropdown.value === "keywords") {
+            numberInput.style.display = "inline";
+            numberLabel.style.display = "inline";
+            numberInput.disabled = false;
+            numberLabel.textContent = "Count: ";
+            questionInput.style.display = "none";
+            questionLabel.style.display = "none";
+            submitButton.disabled = false;
+        } else if (taskDropdown.value === "Q&A") {
+            numberInput.style.display = "none";
+            numberLabel.style.display = "none";
+            questionInput.style.display = "inline";
+            questionLabel.style.display = "inline";
+            submitButton.disabled = false;
+        } else {
+            numberInput.style.display = "none";
+            numberLabel.style.display = "none";
+            questionInput.style.display = "none";
+            questionLabel.style.display = "none";
+            submitButton.disabled = true;
+        }
+    }
+    
+    document.querySelector("select[name='tasks']").addEventListener("change", toggleElements);
+    toggleElements();
+});

static/style.css

@@ -0,0 +1,90 @@
+* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: Arial, sans-serif;
+    background-color: #f4f4f4;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    height: 100vh;
+}
+
+.container {
+    width: 50%;
+    background: white;
+    padding: 30px;
+    box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
+    border-radius: 8px;
+    text-align: center;
+}
+
+h1 {
+    color: #333;
+    margin-bottom: 20px;
+}
+
+label {
+    font-size: 16px;
+    font-weight: bold;
+    display: block;
+    margin-top: 15px;
+    color: #555;
+    text-align: left;
+}
+
+textarea, select, input {
+    width: 100%;
+    padding: 10px;
+    margin-top: 5px;
+    border: 1px solid #ccc;
+    border-radius: 5px;
+    font-size: 16px;
+}
+
+select {
+    cursor: pointer;
+}
+
+button {
+    background: #4CAF50;
+    color: white;
+    padding: 12px 20px;
+    margin-top: 20px;
+    border: none;
+    cursor: pointer;
+    border-radius: 5px;
+    font-size: 16px;
+    width: 100%;
+    transition: background 0.3s ease;
+}
+
+button:disabled {
+    background: #ccc;
+    cursor: not-allowed;
+}
+
+button:hover:not(:disabled) {
+    background: #45a049;
+}
+
+#number_label {
+    display: none;
+}
+
+#number {
+    display: none;
+}
+
+.result-section {
+    margin-top: 20px;
+    padding: 15px;
+    background-color: #eef;
+    border-left: 5px solid #4CAF50;
+    border-radius: 5px;
+    font-size: 16px;
+    text-align: left;
+}

templates/index.html

@@ -0,0 +1,44 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Knowledge Assistant</title>
+    <link rel="stylesheet" href="static/style.css">
+    <script src="static/script.js" defer></script>
+</head>
+<body>
+    <div class="container">
+        <h1>Enter Details</h1>
+        <form action="/" method="POST">
+            <label for="user_text">Enter Your Text: </label>
+            <textarea name="user_text" rows="4" cols="50" required>{{ user_text if user_text else '' }}</textarea>
+            <br><br>
+
+            <label for="tasks">Choose a Task:</label>
+            <select name="tasks" required onchange="toggleElements()">
+                <option value="none">--Select--</option>
+                <option value="summarization" {% if task == 'summarization' %}selected{% endif %}>Summarize Text</option>
+                <option value="keywords" {% if task == 'keywords' %}selected{% endif %}>Get Keywords</option>
+                <option value="Q&A" {% if task == 'Q&A' %}selected{% endif %}>Ask Questions</option>
+            </select>
+            <br><br>
+
+            <label id="number_label" for="user_number" style="display: none;"></label>
+            <input type="number" id="number" name="user_number" style="display: none;">
+            <label id="question_label" for="user_question" style="display: none;">Enter Your Question:</label>
+            <input type="text" id="question" name="user_question" style="display: none;">
+            <br><br>
+
+            <button type="submit" id="submit_btn" disabled>Submit</button>
+        </form>
+
+        {% if text %}
+        <div class="result-section">
+            <h2>Result for {{ task }}:</h2>
+            <p>{{ text }}</p>
+        </div>
+        {% endif %}
+    </div>
+</body>
+</html>

templates/results.html

@@ -0,0 +1,3 @@
+<h1> {{ task }}: </h1>
+<br>
+{{ text }}