Upload 5 files

Uploaded core scripts, knowledge base and files

adversarial_framework.py

@@ -0,0 +1,261 @@
+# adversarial_framework.py
+
+from typing import Literal, Dict, List, Tuple
+from difflib import SequenceMatcher
+from sentence_transformers import SentenceTransformer, util
+from numpy.polynomial.polynomial import Polynomial
+import nlpaug.augmenter.word as naw
+import nltk
+import numpy as np
+import pandas as pd
+import base64
+from datetime import datetime
+from io import BytesIO
+import matplotlib.pyplot as plt
+nltk.download('averaged_perceptron_tagger_eng')
+
+class StatisticalEvaluator:
+    """
+    Computes statistical insights over response similarity scores.
+    Useful for summarizing adversarial robustness.
+    """
+    def __init__(self, scores: List[float]):
+        self.scores = np.array(scores)
+
+    def mean(self) -> float:
+        return round(np.mean(self.scores), 2)
+
+    def median(self) -> float:
+        return round(np.median(self.scores), 2)
+
+    def variance(self) -> float:
+        return round(np.var(self.scores), 2)
+
+    def std_dev(self) -> float:
+        return round(np.std(self.scores), 2)
+
+    def min_score(self) -> float:
+        return round(np.min(self.scores), 2)
+
+    def max_score(self) -> float:
+        return round(np.max(self.scores), 2)
+
+    def summary(self) -> Dict[str, float]:
+        return {
+            "mean": self.mean(),
+            "median": self.median(),
+            "std_dev": self.std_dev(),
+            "variance": self.variance(),
+            "min": self.min_score(),
+            "max": self.max_score(),
+        }
+
+class SimilarityCalculator:
+    def __init__(self, model_name: str = "sentence-transformers/paraphrase-MiniLM-L3-v2"):
+        self.model = SentenceTransformer(model_name)
+
+    def cosine_similarity(self, original: str, perturbed: str) -> float:
+        emb1 = self.model.encode(original, convert_to_tensor=True)
+        emb2 = self.model.encode(perturbed, convert_to_tensor=True)
+        raw_score = util.pytorch_cos_sim(emb1, emb2).item()
+        clamped_score = max(0.0, min(raw_score, 1.0))
+        return round(clamped_score * 100, 2)
+
+    def sequence_similarity(self, original: str, perturbed: str) -> float:
+        return round(SequenceMatcher(None, original, perturbed).ratio() * 100, 2)
+
+class AdversarialRiskCalculator:
+    def __init__(self, alpha: float = 2, beta: float = 1.5):
+        self.alpha = alpha
+        self.beta = beta
+
+    def compute_ari(self, query_sim: float, response_sim: float) -> float:
+        q, r = query_sim / 100, response_sim / 100
+        ari = ((1 - r) ** self.alpha) * ((1 + (1 - q)) ** self.beta)
+        return round(ari * 100, 2)
+
+class PSCAnalyzer:
+    def __init__(self, degree: int = 5, r: int = 10):
+        self.r = r
+        self.degree = degree
+
+    def _bin_data(self, x: np.ndarray, y: np.ndarray, mode='max') -> Tuple[np.ndarray, np.ndarray]:
+        bins = np.linspace(min(x), max(x), self.r + 1)
+        best_x, best_y = [], []
+
+        for i in range(self.r):
+            mask = (x >= bins[i]) & (x < bins[i + 1])
+            sub_x, sub_y = x[mask], y[mask]
+
+            if len(sub_x) > 0:
+                if mode == 'max':
+                    idx = np.argmax(sub_y)
+                elif mode == 'min':
+                    idx = np.argmin(sub_y)
+                else:
+                    raise ValueError("mode must be 'max' or 'min'")
+
+                best_x.append(sub_x[idx])
+                best_y.append(sub_y[idx])
+
+        return np.array(best_x), np.array(best_y)
+
+    def fit_and_auc(self, x, y):
+        x = np.array(x)
+        y = np.array(y)
+
+        coeffs = np.polyfit(x, y, self.degree)
+        poly_fn = np.poly1d(coeffs)
+        fitted_y = poly_fn(x)
+
+        auc_val = round(np.trapz(fitted_y, x), 4)
+        return auc_val, fitted_y
+
+    def plot_curve(self, x: np.ndarray, y: np.ndarray, fitted: np.ndarray, title: str, label: str, save_path=None):
+        plt.figure(figsize=(8, 5))
+        plt.plot(x, y, 'o', label='Sampled Points')
+        plt.plot(x, fitted, '--', label='Fitted Curve')
+        plt.xlabel('Perturbation / Queries')
+        plt.ylabel(label)
+        plt.title(title)
+        plt.legend()
+        plt.grid(True)
+        if save_path:
+            plt.savefig(save_path)
+        plt.show()
+
+    def evaluate(self, x_vals: List[float], y_vals: List[float], mode: str = 'max', label: str = 'ASR Curve') -> float:
+        x, y = self._bin_data(np.array(x_vals), np.array(y_vals), mode=mode)
+        auc_val, fitted = self.fit_and_auc(x, y)
+        self.plot_curve(x, y, fitted, title=f"PSC-{label}", label=label)
+        return auc_val
+
+    def run_psc_analysis(self, x_vals: List[float], y_vals: List[float], save_csv: str ="psc_export.csv", plot : bool = True):
+        auc = self.evaluate(x_vals, y_vals, mode="max", label="Semantic Similarity" if plot else "")
+        df = pd.DataFrame({"perturbation_level": x_vals, "response_similarity": y_vals})
+        df.to_csv(save_csv, index=False)
+        print(f"📉 PSC-AUC: {auc} | 📁 CSV saved to: {save_csv}")
+        return auc
+
+class TextPerturber:
+    def __init__(self):
+        self.methods = {
+            "synonym": naw.SynonymAug(aug_src='wordnet'),
+            "delete": naw.RandomWordAug(action="delete"),
+            "contextual": naw.ContextualWordEmbsAug()
+        }
+
+    def perturb(self, input_text: str, aug_method: Literal["synonym", "delete", "contextual"] = "synonym") -> str:
+        if aug_method not in self.methods:
+            raise ValueError(f"Invalid method '{aug_method}'. Choose from {list(self.methods.keys())}.")
+        result = self.methods[aug_method].augment(input_text)
+        return result[0] if isinstance(result, list) else result
+
+class AdversarialAttackPipeline:
+    def __init__(self, answer_generator):
+        self.similarity = SimilarityCalculator()
+        self.risk_calculator = AdversarialRiskCalculator()
+        self.perturber = TextPerturber()
+        self.answer_generator = answer_generator
+
+    def run(self, query: str, top_k: int = 3, perturb_method: str = "synonym") -> Dict:
+        normal_response = self.answer_generator(query, top_k)
+        perturbed_query = self.perturber.perturb(query, perturb_method)
+        adversarial_response = self.perturber.perturb(normal_response, perturb_method)
+        perturbed_response = self.answer_generator(perturbed_query, top_k)
+
+        cos_metrics = {
+            "query_sim": self.similarity.cosine_similarity(query, perturbed_query),
+            "adversarial_sim": self.similarity.cosine_similarity(normal_response, adversarial_response),
+            "response_sim": self.similarity.cosine_similarity(normal_response, perturbed_response),
+        }
+
+        seq_metrics = {
+            "query_seq_match": self.similarity.sequence_similarity(query, perturbed_query),
+            "adv_seq_match": self.similarity.sequence_similarity(normal_response, adversarial_response),
+            "resp_seq_match": self.similarity.sequence_similarity(normal_response, perturbed_response),
+        }
+
+        ari = self.risk_calculator.compute_ari(cos_metrics['query_sim'], cos_metrics['response_sim'])
+
+        self._print_report(query, normal_response, perturbed_query, perturbed_response, adversarial_response, cos_metrics, seq_metrics, ari)
+
+        return {
+            "normal_query": query,
+            "normal_response": normal_response,
+            "perturbed_query": perturbed_query,
+            "perturbed_response": perturbed_response,
+            "adversarial_response": adversarial_response,
+            "cos_sim": cos_metrics,
+            "seq_match": seq_metrics,
+            "ari": ari,
+        }
+
+    def _print_report(self, query, normal, pert_q, pert_r, adv_r, cos, seq, ari):
+        print("🔵 Original Query:", query)
+        print("\n🟢 Normal Response:", normal)
+        print("\n🔴 Direct Perturbation of Generated Response:", adv_r)
+        print("\n🟠 Perturbed Query:", pert_q)
+        print("\n🔴 Perturbed Response:", pert_r)
+        print(f"\n📊 Cosine Sim — Perturbed Query: {cos['query_sim']}%, Adversarial: {cos['adversarial_sim']}%, Perturbed Response: {cos['response_sim']}%")
+        print(f"\n📊 Seq Match — Perturbed Query: {seq['query_seq_match']}%, Adversarial: {seq['adv_seq_match']}%, Perturbed Response: {seq['resp_seq_match']}%")
+        print(f"\n🔺 ARI (Adversarial Risk Index): {ari}")
+
+    def plot_to_base64(self, fig):
+        """
+        Converts a Matplotlib figure to a base64-encoded image string.
+        Useful for sending plots in web apps or saving as embeddable outputs.
+        """
+        buf = BytesIO()
+        fig.savefig(buf, format='png')
+        buf.seek(0)
+        image_base64 = base64.b64encode(buf.read()).decode('utf-8')
+        buf.close()
+        return f"data:image/png;base64,{image_base64}"
+
+    def evaluate_adversarial_robustness(self, query, method, k, psc_degree: int = 4, 
+                                        ep_min: float = 0.1, ep_max: float = 4.1, 
+                                        ep_gap: float = 0.2):
+        """
+        Evaluate semantic robustness of the pipeline over increasing perturbation intensities.
+        Added as instance method so pipeline context (e.g. answer generator) does not need to be recreated.
+        """
+        epsilons = np.arange(ep_min, ep_max, ep_gap)
+        x_vals, y_vals, ari_vals = [], [], []
+
+        for epsilon in epsilons:
+            result = self.run(query=query, top_k=k, perturb_method=method)
+            x_vals.append(round(epsilon, 2))
+            y_vals.append(result['cos_sim']['response_sim'])
+            ari_vals.append(result['ari'])
+
+        auc = PSCAnalyzer(degree=psc_degree, r=10).evaluate(x_vals, y_vals, mode="max", label="Semantic Similarity")
+
+        stats = StatisticalEvaluator(y_vals).summary()
+        stats_text = "\n".join([f"{k}: {v}" for k, v in stats.items()])
+
+        # Save CSV with full results and stats
+        df = pd.DataFrame({
+            "Perturbation_Level": x_vals,
+            "Response_Similarity": y_vals,
+            "ARI": ari_vals
+        })
+        summary_df = pd.DataFrame.from_dict(stats, orient='index', columns=["Response_Similarity_Stats"])
+        summary_df.reset_index(inplace=True)
+        summary_df.rename(columns={"index": "Metric"}, inplace=True)
+        export_df = pd.concat([df, pd.DataFrame([{}]), summary_df], ignore_index=True)
+        export_df.to_csv("gradio_output.csv", index=False)
+
+        coeffs = np.polyfit(x_vals, y_vals, psc_degree)
+        poly_fn = np.poly1d(coeffs)
+        fitted = poly_fn(x_vals)
+        fig, ax = plt.subplots()
+        ax.plot(x_vals, y_vals, 'o', label='Sampled Points')
+        ax.plot(x_vals, fitted, '--', label='Fitted Curve')
+        ax.set_xlabel('Perturbation Level (Epsilon)')
+        ax.set_ylabel('Semantic Similarity')
+        ax.set_title('Perturbation Sensitivity Curve (PSC)')
+        ax.legend()
+        ax.grid(True)
+        
+        return stats_text, auc, fig

app.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""FULL WORKFLOW
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1I9KysFRYpSZfkL7FIxbodBYUgudQRejA
+"""
+
+
+
+!pip install transformers faiss-cpu gradio sentence-transformers nlpaug scikit-learn
+
+# rag_module.py
+
+import json
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from sklearn.feature_extraction.text import TfidfVectorizer
+from transformers import AutoTokenizer, T5ForConditionalGeneration
+import gradio as gr
+
+from rag_pipeline import RAGPipeline
+
+from adversarial_framework import *
+
+# Example Usage:
+rag = RAGPipeline()
+#print(rag.generate_answer("Who is the president of NACOS?"))
+
+
+
+adv_pipeline = AdversarialAttackPipeline(answer_generator=rag.generate_answer)
+
+# Gradio function
+def gradio_wrapper(query, method, k):
+    stats_text, auc, fig, pert_q, pert_r, adv_r = adv_pipeline.evaluate_adversarial_robustness(
+        query=query,
+        method=method,
+        k=k
+    )
+
+    return (
+        stats_text,
+        f"{auc}",
+        fig,
+        f"🟠 Perturbed Query:\n\n{pert_q}",
+        f"🟢 Perturbed Response:\n\n{pert_r}",
+        f"🔴 Directly Perturbed Response of Normal Output:\n\n{adv_r}"
+    )
+
+# Build Interface
+gr.Interface(
+    fn=gradio_wrapper,
+    inputs=[
+        gr.Textbox(label="Enter a Question"),
+        gr.Dropdown(choices=["synonym", "delete", "contextual"], label="Perturbation Method"),
+        gr.Slider(1, 5, step=1, value=3, label="Top-K Retrieved Chunks")
+    ],
+    outputs=[
+        gr.Textbox(label="📊 Summary Statistics"),
+        gr.Textbox(label="🔺 PSC-AUC Score"),
+        gr.Plot(label="📈 PSC Curve"),
+        gr.Textbox(label="🟠 Perturbed Query Example"),
+        gr.Textbox(label="🟢 Perturbed Response Example"),
+        gr.Textbox(label="🔴 Directly Perturbed Normal Response Example")
+    ],
+    title="Adversarial Testing on RAGiant System",
+    description="Evaluate robustness against textual attacks and visualize degradation with ARI & PSC-AUC."
+).launch()
+
+
+

calebdata.json

@@ -0,0 +1,872 @@
+[
+    {
+        "chunk_id": "AVMV-000",
+        "title": "Full Anthem Lyrics",
+        "section": "Anthem, Values, Mission, Vision",
+        "text": "Hail the Institution, Great!\nSo brave and full of faith,\nFor God and for Humanity,\nAn awesome Varsity.\nThere is peace within its walls,\nAnd joy and blessedness,\nMoral values are its stalls,\nWith love and faithfulness.\nFor the good LORD is our strength,\nOur song and salvation,\nDivine wisdom is thy breath,\nOur shield and solution.\nHail the great institution, CALEB!\nCitadel of knowledge!\nGreat and brave CALEB!\nFor one and for all.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "history",
+                "foundation",
+                "visitor",
+                "anthem",
+                "school anthem"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "AVMV-001",
+        "title": "University Mission Statement",
+        "section": "Anthem, Values, Mission, Vision",
+        "text": "To provide globally competitive education and research in a godly environment with visible and positive societal impact.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "mission",
+                "values",
+                "mission statement"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "AVMV-002",
+        "title": "University Vision Statement",
+        "section": "Anthem, Values, Mission, Vision",
+        "text": "To be an innovative leader in producing impactful human resources, while sustaining excellence in learning, service, and godly character.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "vision",
+                "values",
+                "vision statement"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "AVMV-003",
+        "title": "Seven Core Values",
+        "section": "Anthem, Values, Mission, Vision",
+        "text": "Core Values: There are 7 seven which are:\n1 Excellence \u00e2\u20ac\u201c Upholding academic and professional rigor.\n2 Integrity \u00e2\u20ac\u201c Ensuring honesty, transparency, and accountability.\n3 Service \u00e2\u20ac\u201c Embracing selflessness to uplift society.\n4 Innovation \u00e2\u20ac\u201c Fostering creativity and adaptive problem-solving.\n5 Inclusivity \u00e2\u20ac\u201c Valuing diversity and creating a welcoming environment.\n6 Leadership \u00e2\u20ac\u201c Nurturing ethical, visionary, and compassionate leaders.\n7 Community \u00e2\u20ac\u201c Encouraging collaboration, mutual respect, and support.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "foundation",
+                "values",
+                "seven core values"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "IOV-000",
+        "title": "University Background and Licensing",
+        "section": "Institutional Overview",
+        "text": "Caleb University, located in Imota, Lagos, received its provisional operating license from the Federal Government of Nigeria on May 17, 2007. The institution is owned by Dr. Ola Adebogun. Dr. Ola Adebogun serves as the Chairman and Visitor.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "history",
+                "foundation",
+                "relocation",
+                "location",
+                "background",
+                "established",
+                "founded",
+                "tell me about Caleb"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "IOV-001",
+        "title": "Program Approvals Timeline",
+        "section": "Institutional Overview",
+        "text": "The University officially commenced academic activities on January 21, 2008, admitting its pioneer students into three colleges. \n\nSince 2011, Caleb University has graduated eleven cohorts, all of whom have been successfully mobilized for the mandatory one-year National Youth Service Corps (NYSC).\n\nIn 2012, the National Universities Commission (NUC) approved Part-Time programmes under the Caleb Business School (CBS), as well as the establishment of the College of Postgraduate Studies (COPOS), which launched its first M.Sc. Architecture programme in March 2013. The College of Education (COLED) received NUC approval in August 2018, while the College of Law (COLAW) was accredited by both the NUC and the Council for Legal Education in 2021.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "legal approval",
+                "commencement",
+                "relocation"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "IOV-002",
+        "title": "Relocation & Permanent Site",
+        "section": "Institutional Overview",
+        "text": "Initially operating from its approved take-off site in Ikosi GRA, Lagos, the University relocated in November 2009 to its permanent 110-hectare campus situated at Km 15, Ikorodu-Itoikin Road, Imota, Lagos.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "history",
+                "establishment",
+                "relocation",
+                "location",
+                "how big is the Caleb land?"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "IOV-003",
+        "title": "More About Caleb",
+        "section": "Institutional Overview",
+        "text": "Caleb University is a center of excellence in teaching, learning, and research. Frequently ranked among the top 10 most sought-after private universities in Nigeria, CUL is recognized for its innovation and ability to transform ideas into impactful solutions. \nSince its inception, the institution has invested in nurturing individuals with curiosity, empathy, drive, and purpose. \nStudents of Caleb University are trained to be globally competitive while maintaining strong moral character and demonstrable academic excellence. \n Caleb University prioritizes discipleship work, fostering spiritual growth through chapel services, Bible studies, and discipleship programs. These initiatives encourage students to deepen their faith and develop character traits like integrity and compassion. Faculty and staff serve as mentors, guiding students in spiritual formation and ethical leadership. By integrating faith into the academic experience, Caleb equips students to live purposefully, embodying Christian values and making a positive impact in society.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "recognition",
+                "foundation",
+                "about",
+                "overview",
+                "rating",
+                "ranking",
+                "tell me about Caleb"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "GBT-000",
+        "title": "Visitor",
+        "section": "Gov. & Board of Trustees",
+        "text": "Governance, Board of Trustees",
+        "metadata": {
+            "source": "The Visitor, in Caleb University's case, Dr. Oladega Adebogun, is the Proprietor and Managing Director/CEO of Caleb Group of Schools. He also functions as an advisor and guide, ensuring the university adheres to its mission and values.\n The current Visitor is Dr. Oladega Adebogun",
+            "tags": [
+                "Caleb University Profile"
+            ],
+            "college": "governance;owner;visitor;proprietor",
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "GBT-001",
+        "title": "Chairman",
+        "section": "Gov. & Board of Trustees",
+        "text": "Governance, Board of Trustees",
+        "metadata": {
+            "source": "Prince Abiodun Ogunleye",
+            "tags": [
+                "Caleb University Profile"
+            ],
+            "college": "governance;chairman",
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "GBT-002",
+        "title": "Pro-Chancellor",
+        "section": "Gov. & Board of Trustees",
+        "text": "Governance, Board of Trustees",
+        "metadata": {
+            "source": "Prof. Fola Tayo",
+            "tags": [
+                "Caleb University Profile"
+            ],
+            "college": "board of trustees;pro-chancellor",
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "GBT-003",
+        "title": "Chancellor",
+        "section": "Gov. & Board of Trustees",
+        "text": "Governance, Board of Trustees",
+        "metadata": {
+            "source": "The Chancellor is a ceremonial head of the university, representing the institution at important events and advocating for its interests. The Chancellor may also play a role in fundraising and strategic planning. \n The current Chancellor is Chief (Dr.) Edwin K. Clark, OFR, CON",
+            "tags": [
+                "Caleb University Profile"
+            ],
+            "college": "governance;ceremonial head;chancellor",
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "MPO-000",
+        "title": "Vice-Chancellor",
+        "section": "Mgt. & Principal Officers",
+        "text": "The VC leads the university with strategic vision and direction, overseeing the implementation of academic policies and research initiatives. He/she plays a pivotal role in advancing the university's mission.\n The current Vice Chancellor is Prof. Nosa Owens-Ibie",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "management",
+                "principal officers",
+                "VC",
+                "vc",
+                "vice-chancellor"
+            ],
+            "college": null,
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "MPO-001",
+        "title": "Deputy Vice Chancellors",
+        "section": "Mgt. & Principal Officers",
+        "text": "There are two Deputy Vice Chancellors; DVC-RISA (Deputy Vice-Chancellor Research, Innovation, Strategy & Administration) Prof. Olalekan Asikhia & DVC-ACAD (Deputy Vice-Chancellor Academics) Prof. Sunday Adewale",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "deputy vice-chancellor",
+                "dvc",
+                "DVC",
+                "who is the DVC of Caleb University?",
+                "who is the Deputy Vice Chancellor of Caleb University?"
+            ],
+            "college": null,
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "MPO-002",
+        "title": "Deputy Vice Chancellor (Research, Innovation, Strategy & Administration)",
+        "section": "Mgt. & Principal Officers",
+        "text": "The DVC RISA spearheads research excellence and strategic partnerships, driving innovation both within and beyond the university. Under his guidance, Caleb University aims to enhance its research capabilities and contribute to knowledge creation. \n The current DVC-RISA is Prof. Olalekan Asikhia",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "deputy vice-chancellor RISA",
+                "dvc",
+                "DVC",
+                "dvc-risa",
+                "dvc risa"
+            ],
+            "college": null,
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "MPO-003",
+        "title": "Deputy Vice Chancellor (Academics)",
+        "section": "Mgt. & Principal Officers",
+        "text": "The DVC ACAD is responsible for academic affairs and student welfare, ensuring the delivery of high-quality education. This leadership focuses on promoting academic excellence and student-centered learning approaches.\nThe current DVC-Acad and Deputy Vice Chancellor is Prof. Sunday Adewale",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "deputy vice-chancellor ACAD",
+                "dvc",
+                "DVC",
+                "dvc-acad",
+                "dvc acad"
+            ],
+            "college": null,
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "MPO-004",
+        "title": "Registrar",
+        "section": "Mgt. & Principal Officers",
+        "text": "The Registrar serves as the chief administrative officer, managing university records, admissions, and examinations. With meticulous attention to detail, he/she ensures the integrity and efficiency of administrative processes. \nThe current Registrar is Mr. Mayokun Olumeru",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "registrar"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "MPO-005",
+        "title": "Bursar",
+        "section": "Mgt. & Principal Officers",
+        "text": "The Bursar oversees financial management and resource allocation, maintaining fiscal discipline and transparency. This stewardship contributes to the financial sustainability of the university. The current Bursar is Mr. Adesina Abubakre",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "bursar"
+            ],
+            "college": null,
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "MPO-006",
+        "title": "Librarian",
+        "section": "Mgt. & Principal Officers",
+        "text": "The Librarian leads library services, providing access to academic resources and supporting scholarly communication. The librarian works to foster a dynamic learning environment that encourages intellectual exploration on campus. The current librarian is Mr. Josiah Adeyomoye",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "librarian"
+            ],
+            "college": null,
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "MPO-007",
+        "title": "University Governance Structure",
+        "section": "Mgt. & Principal Officers",
+        "text": "Caleb University operates a three-tier management structure consisting of the Board of Trustees, the Governing Council, and the University Management which is also known as pricipal officers.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "university management"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "MPO-008",
+        "title": "Executive Positions in Governance",
+        "section": "Mgt. & Principal Officers",
+        "text": "The Visitor is Dr. Oladega Adebogun with Board of Trustees is chaired by Prince Abiodun Ogunleye, while the Governing Council is chaired by Prof. Fola Tayo, who also serves as the Pro-Chancellor. ",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "board of trustees",
+                "governing council",
+                "management"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "MPO-009",
+        "title": "Principal Officers also known as University Management",
+        "section": "Mgt. & Principal Officers",
+        "text": "The University Management is led by six Principal Officers: Prof. Nosa Owens-Ibie (Vice Chancellor), Prof. Olalekan Asikhia (DVC \u00e2\u20ac\u201c Research, Innovation, Strategy & Admin), Prof. Sunday Adewale (DVC \u00e2\u20ac\u201c Academics), Mr. Mayokun Olumeru (Registrar), Mr. Adesina Abubakre (Bursar), and Mr. Josiah Adeyomoye (University Librarian).",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "principal officers",
+                "management",
+                "university management"
+            ],
+            "college": null,
+            "type": "about"
+        }
+    },
+    {
+        "chunk_id": "ACA-000",
+        "title": "Colleges & Faculties",
+        "section": "Academic Programmes",
+        "text": "Caleb has over 30 undergraduate degree programmes and 10+ postgraduate degree programmes across seven colleges: COPAS, CONBAMS, COLENSMA, CASMAS, COPOS, COLAW, COLED. All mature programmes/courses are accredited by the NUC and respective professional bodies.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programs",
+                "programmes",
+                "departments",
+                "programs",
+                "colleges"
+            ],
+            "college": "all",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-001",
+        "title": "Offices & Positions in Colleges",
+        "section": "Academic Programmes",
+        "text": "Dean: Every college has a dean of College that sits as the presiding office usually over all college affairs. \nDeputy Dean: This individual assists the Dean in decision-making as well as execution. There is no dedicated physical office for this role though.\nHead of Department (HOD): Every department has a head that coordinates activities within the department unit. This individual oversees the activities of students and is the directly above lecturers in the pecking order. His/her approval is also needed in many official cases\nCollege Secretary: This person is concerned with the taking down minutes in College wide meetings, holding the records of personnel in a College amongst other things. \n Department Secretary: This individual is saddled with the responsiblity of assisting at the Department office, ensuring all records are well kept. /n Lecturer: is the individual that pours out wealth of experience and knowledge into students in the department. /nLevel Adviser/Advisor: this individual handles student queries and answers questions as relating to grades, courses and wellbeing of the general student populace in a level",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "college posts",
+                "positions",
+                "colleges"
+            ],
+            "college": "all",
+            "type": "staff"
+        }
+    },
+    {
+        "chunk_id": "ACA-002",
+        "title": "Programs under COPAS",
+        "section": "Academic Programmes",
+        "text": "This college cultivates scientific curiosity and practical knowledge through diverse programs in life, physical, and computing sciences. Students engage in cutting-edge research under experienced faculty. \nThe College of Pure and Applied Sciences (COPAS) offers eight undergraduate programs: B.Sc. Biochemistry, B.Sc. Computer Science, B.Sc. Cyber Security, B.Sc. Environmental Management and Toxicology, B.Sc. Industrial Chemistry, B.Sc. Information Systems, B.Sc. Microbiology and Industrial Biotechnology, B.Sc. Software Engineering.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "science",
+                "COPAS"
+            ],
+            "college": "COPAS",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-003",
+        "title": "Programs/Courses under CASMAS",
+        "section": "Academic Programmes",
+        "text": "CASMAS equips students with a blend of analytical, creative, and managerial skills for societal transformation and economic development. The College fosters critical thinking and innovation. \nThe College of Social and Management Sciences (CASMAS) offers twelve undergraduate programs: B.Sc. Accounting, B.Sc. Taxation, B.Sc. Banking and Finance, B.Sc. Business Administration (Marketing, General Business, HRM), B.Sc. Criminology and Security Studies, B.Sc. Economics, B.Sc. International Relations, B.Sc. Mass Communication (Print, PR & Advertising, Broadcast, Film & Cinematography), B.Sc. Peace Studies and Conflict Resolution, B.Sc. Political Science, B.Sc. Public Administration, B.Sc. Psychology",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "arts",
+                "CASMAS"
+            ],
+            "college": "CASMAS",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-004",
+        "title": "Programs under COLENSMA",
+        "section": "Academic Programmes",
+        "text": "COLENSMA trains professionals in sustainable design and environmental management. With practical projects and expert faculty, students are equipped for a future in architecture, urban planning, and more.\nThe College of Environmental Sciences & Management (COLENSMA) offers three undergraduate programs/courses/departments: B.Sc./M.Sc. Architecture (6-year program), B.Sc. Estate Management, B.Sc. Quanitity Surveying",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "architecture",
+                "estate",
+                "COLENSMA"
+            ],
+            "college": "COLENSMA",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-005",
+        "title": "Programs under COLAW",
+        "section": "Academic Programmes",
+        "text": "COLAW is committed to producing principled legal professionals. With a strong academic foundation and ethical orientation, students are groomed for leadership in legal practice and governance. \nThe College of Law (COLAW) offers two programs: LLB/LLM Public Law, LLB/LLM Private and Property Law",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "law",
+                "legal",
+                "COLAW"
+            ],
+            "college": "COLAW",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-006",
+        "title": "Programs under CONBAMS",
+        "section": "Academic Programmes",
+        "text": "This college produces world-class nurses and health professionals through a curriculum focused on clinical excellence and global health standards. \nThe College of Nursing and Basic Medical Sciences (CONBAMS) offers three programs: B.N.Sc. Nursing Science, B.Sc. Human Anatomy, B.Sc. Human Physiology",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "nursing",
+                "medicine",
+                "science",
+                "CONBAMS"
+            ],
+            "college": "CONBAMS",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-007",
+        "title": "Programs under COPOS",
+        "section": "Academic Programmes",
+        "text": "COPOS offers advanced degrees across various disciplines, with faculty members who are seasoned researchers. The College supports innovative research and professional growth. \nThe College of Postgraduate Studies (COPOS) offers nine programs: Architecture (Ph.D, M.Sc), Accounting, Taxation & Finance (M.Phil./Ph.D, M.Sc, PGD, MBA options), Economics (M.Phil./Ph.D, M.Sc, PGD), Computer Science (M.Phil./Ph.D, M.Sc, PGD), Mass Communication (M.Phil./Ph.D, M.Sc, PGD), Business Administration [MBA Specializations] (HR, Marketing, General, International Business, Management), Political Science & International Relations (M.Phil./Ph.D, M.Sc, PGD), Microbiology & Biochemistry (M.Phil./Ph.D, M.Sc, PGD), Christian Religious Studies (M.Phil./Ph.D, M.Ed, PGDE)",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "masters",
+                "postgraduate",
+                "COPOS"
+            ],
+            "college": "COPOS",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ACA-008",
+        "title": "Programs under COLED",
+        "section": "Academic Programmes",
+        "text": "The College of Education (COLED) has a couple of departments, but more inquires can be made by contacting.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programmes",
+                "departments",
+                "education",
+                "COLED"
+            ],
+            "college": "COLED",
+            "type": "academic"
+        }
+    },
+    {
+        "chunk_id": "ORG-000",
+        "title": "SRC",
+        "section": "Organizations",
+        "text": "The Student Representative Council (SRC) is a student led group that is saddled with representing student opinions and as well as relating pain points to the Vice Chancellor and Management. It is the variant of what many in public schools would refer to as the Student Union. The current President is David, and Vice-President Mari Omoge. Leaders are delegated based on multiple criteria, and not by open balloting",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "organizations",
+                "student union"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "ORG-001",
+        "title": "CUAPASS",
+        "section": "Organizations",
+        "text": "CUAPASS stands for Caleb University of Pure and Applied Sciences. It comprises of all students in COPAS. Presidency is based on the results of yearly polls.",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "organizations",
+                "College of Pure and Applied Sciences",
+                "copas"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "ORG-002",
+        "title": "NACOS",
+        "section": "Organizations",
+        "text": "NACOS stands for NAtional Association of Computing Students.NACOS Caleb Chapter is an association made up of students studying Computer Science, Information Technology, & Software Engineering as well as any other Computing related field. \nIt has the goal to serve both professional and public interests by fostering the open interchange of information and by promoting the highest professional and ethical standards. It organizes seminars, symposia, webinars, computing contests, and other activities aimed at achieving this goal./n The current president of NACOS is Chibuzor Nwachukwu, and the Vice Presidents are Damilola (VP National) and Daniel (VP International). Elections and activities are carried out in accordance to the provisions of the NACOS Consistution ",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "organizations",
+                "computer society",
+                "computer science",
+                "NACOS"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "ORG-003",
+        "title": "GDGoC",
+        "section": "Organizations",
+        "text": "GDGoC Caleb Chapter stands for Google Developer Group, on-Campus Caleb University Chapter (formerly known as GDSC Caleb Chapter). It is an association backed by Google and is saddled with the responsibility of driving tech initiatives irrespective of department barriers. With tracks like data, Cybersecurity, Networking, IOT, it is poised to place Caleb at the frontiers of technological advancement. The current Lead is EFOD Freda, flanked by ADEBULE John. Leadership is conferred by GDG group based on certain qualifications and a rigorous interview procedure organized by Google.",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "organizations",
+                "GDG",
+                "GDSC",
+                "GDGoC"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "ORG-004",
+        "title": "CSEAN",
+        "section": "Organizations",
+        "text": null,
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-000",
+        "title": "About",
+        "section": "The Master's Chapel",
+        "text": "The school's church, The Master's Chapel is a vital part of Caleb University, complementing a student\u00e2\u20ac\u2122s academic and spiritual experience, and providing the campus community with a dynamic worship experience. Every Monday, Wednesday, and Friday, students, faculty, and staff gather for a time centered on the expositional preaching of Scripture. Speakers include chaplains, pastors, missionaries, faculty members, Christian leaders, and students.",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "church",
+                "prayer",
+                "chapel",
+                "TMC"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "TMC-001",
+        "title": "Leadership",
+        "section": "The Master's Chapel",
+        "text": "The Chaplaincy consists of the Chaplain and The Assistant. Currently, these two positions are occupied by Pastor Paul Adeboyega and Pastor Chibuzor respectively. Every session, student leaders known as executives, spearheaded by the General Co-ordinator (also known as G.C) are elected by the leading of The Holy Spirit to assist the Chaplaincy as God ministers to His people.",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "church",
+                "chapel",
+                "TMC",
+                "executives",
+                "chaplaincy"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "TMC-002",
+        "title": "The Chapliancy",
+        "section": "The Master's Chapel",
+        "text": "The Chaplain (also known as Pastor) provides spiritual guidance and support to students and staff, promoting religious activities and fostering a sense of community. This role involves organizing religious services, offering pastoral care, counselling, and promoting ethical behavior in line with the university's values. /nThe Assistant Chaplain supports the Chaplain in their duties, as they both ensure that the religious needs of the university community are met",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "church",
+                "chapel",
+                "TMC",
+                "chaplaincy"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "TMC-003",
+        "title": "Worship Schedules",
+        "section": "The Master's Chapel",
+        "text": "Worship meetings and gatherings takes place on Monday, Wednesday, Friday, and Saturday all with varying times each day, mostly determined by the Student Affairs Officer and/or The Chaplaincy. On Sundays, Mondays, Tuesdays and Wednesdays, services are either based on hall of residence or level. \n Sunday meetings typically start by 07:00, with 3-4 services held. \n On Mondays & Wednesdays respectively, two consecutive worship services (typically 1hr30mins long) are held, between 17:00 and 20:00.  On Tuesdays, Discipleship meeting, between 17:00 and 18:30. On the other days, services and meetings are scheduled for workers alone as well as disiples.",
+        "metadata": {
+            "source": "Research and Observation",
+            "tags": [
+                "church",
+                "chapel",
+                "TMC",
+                "schedule",
+                "time",
+                "meeting",
+                "gathering"
+            ],
+            "college": null,
+            "type": "internal"
+        }
+    },
+    {
+        "chunk_id": "TMC-004",
+        "title": "Executives",
+        "section": "The Master's Chapel",
+        "text": "Young students are selected to lead in various capacities and over the 11 departments, from being hostel representatives, to Heads of Ushering, Protocol, Choir, Media, Sanctuary, Prayer departments etc",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-005",
+        "title": "Overview of Departments",
+        "section": "The Master's Chapel",
+        "text": "There are 10 departments altogether in the Master's Chapel Workforce. Choir who go by the name Glorious Ministers, Protocol, Ushering, Prayer, Evangelism, Devotional Leaders, Sanctuary, Library, Accounting, Media, Drama who go by the name Royal Family",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-006",
+        "title": "Protocol Department",
+        "section": "The Master's Chapel",
+        "text": "They are saddled with the reponsibility of ensuring compliance during the service, which allows for the flow of God's Word and the express move of His Spirit. In some instances, they seize phones and give warnings to those attempting to distrupt the services",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-007",
+        "title": "Ushering Department",
+        "section": "The Master's Chapel",
+        "text": "They are workers that ensure that the congragation flows well into the service as well as are ministered to in terms of offering and tithe envelops, catering and so on. They work with the Protocol officers on some occasions.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-008",
+        "title": "Choir Department",
+        "section": "The Master's Chapel",
+        "text": "These are minsters that carry God's presence and power into the service with songs. Through their Spirit-filled sound, they are poised to lift burdens as well as break the chains of attendees.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-009",
+        "title": "Media Department",
+        "section": "The Master's Chapel",
+        "text": "The Media Department is concerned with displaying the sermon notes, scriptures, messages, announcements and other things digital mainly using a projector or via social media channels. They also are skilled in the arts of recording and making sure highlights of the service are well captured.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-010",
+        "title": "Evangelism Department",
+        "section": "The Master's Chapel",
+        "text": "Following the matching orders of the Lord Jesus Christ, The Evangelism department is ever keen on making sure that no stone is left untouched in the bid to bring salvation on campus. Through earnest prayers and teachings, the heart of the Lord unfolds as well as His desire to restore the strayed sheep.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-011",
+        "title": "Prayer Department",
+        "section": "The Master's Chapel",
+        "text": "The Prayer Department focuses on travailing on bended knees for the campus, its staff, students and Nigeria as a whole. Their goal is that God's will be established upon the earth's face. ",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-012",
+        "title": "Sanctuary Department",
+        "section": "The Master's Chapel",
+        "text": "The Sanctuary Department takes care of the chapel's appearance. Wth a focus on making sure that the holiness of God reflects even in every service with order from arranging chairs to cleaning the venue.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-013",
+        "title": "Library Department",
+        "section": "The Master's Chapel",
+        "text": "These are workers that encourage, enable and drive workers, attendees and members to grow muscles in their mind through the study of the scriptures as well as other value-packed books.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "TMC-014",
+        "title": "Devotional Leaders",
+        "section": "The Master's Chapel",
+        "text": "The devotional leaders group is more like a secondary department in that, members of other are permitted to join in the collective goal of reaching souls across all the hostels and halls of residence.",
+        "metadata": {
+            "source": null,
+            "tags": [
+                "nan"
+            ],
+            "college": null,
+            "type": null
+        }
+    },
+    {
+        "chunk_id": "ACA-000",
+        "title": "Colleges & Faculties",
+        "section": "Department of Computer Science ",
+        "text": "Caleb has over 30 undergraduate degree programmes and 10+ postgraduate degree programmes across seven colleges: COPAS, CONBAMS, COLENSMA, CASMAS, COPOS, COLAW, COLED. All mature programmes/courses are accredited by the NUC and respective professional bodies.",
+        "metadata": {
+            "source": "Caleb University Profile",
+            "tags": [
+                "programs",
+                "courses",
+                "programs",
+                "programmes",
+                "departments",
+                "programs",
+                "colleges"
+            ],
+            "college": "",
+            "type": ""
+        }
+    }
+]

rag_pipeline.py

@@ -0,0 +1,80 @@
+import json
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from sklearn.feature_extraction.text import TfidfVectorizer
+from transformers import AutoTokenizer, T5ForConditionalGeneration
+
+class RAGPipeline:
+    def __init__(
+        self,
+        json_path: str = "calebdata.json",
+        embedder_model: str = "infly/inf-retriever-v1-1.5b",
+        reranker_model: str = "cross-encoder/ms-marco-MiniLM-L-6-v2",
+        generator_model: str = "google/flan-t5-base"
+    ):
+        self.chunks = self._load_chunks(json_path)
+        self.texts = list(set([chunk["text"] or "" for chunk in self.chunks]))
+
+        self.embedder = SentenceTransformer(embedder_model)
+        self.reranker = CrossEncoder(reranker_model)
+        self.tokenizer = AutoTokenizer.from_pretrained(generator_model)
+        self.generator = T5ForConditionalGeneration.from_pretrained(generator_model)
+
+        self.index = self._build_faiss_index()
+        self.tfidf_vectorizer, self.tfidf_matrix = self._build_tfidf()
+
+    def _load_chunks(self, path):
+        with open(path, "r") as f:
+            return json.load(f)
+
+    def _build_faiss_index(self):
+        embeddings = self.embedder.encode(self.texts, convert_to_numpy=True)
+        dimension = embeddings.shape[1]
+        index = faiss.IndexFlatL2(dimension)
+        index.add(embeddings)
+        return index
+
+    def _build_tfidf(self):
+        vectorizer = TfidfVectorizer()
+        matrix = vectorizer.fit_transform(self.texts)
+        return vectorizer, matrix
+
+    def _rerank(self, query, docs):
+        pairs = [(query, doc) for doc in docs]
+        scores = self.reranker.predict(pairs)
+        return [doc for _, doc in sorted(zip(scores, docs), reverse=True)]
+
+    def hybrid_search(self, query, top_k=3):
+        query_embedding = self.embedder.encode([query])[0]
+        _, faiss_indices = self.index.search(np.array([query_embedding]), top_k)
+        faiss_results = [self.texts[i] for i in faiss_indices[0]]
+
+        query_tfidf = self.tfidf_vectorizer.transform([query])
+        tfidf_scores = np.array(query_tfidf.dot(self.tfidf_matrix.T).toarray()).flatten()
+        tfidf_indices = tfidf_scores.argsort()[-top_k:][::-1]
+        tfidf_results = [self.texts[i] for i in tfidf_indices]
+
+        combined = list(set(faiss_results + tfidf_results))
+        return self._rerank(query, combined)[:top_k]
+
+    def generate_answer(self, query, top_k=3):
+        context = "\n".join(self.hybrid_search(query, top_k))
+        prompt = f"Answer the question based on the context.\n\nContext:\n{context}\n\nQuestion: {query}\nAnswer:"
+        inputs = self.tokenizer(prompt, return_tensors="pt", truncation=True, padding=True, max_length=512)
+        output = self.generator.generate(
+            input_ids=inputs["input_ids"],
+            attention_mask=inputs["attention_mask"],
+            max_length=300,
+            do_sample=True,
+            top_p=0.95,
+            top_k=50,
+            pad_token_id=self.tokenizer.eos_token_id
+        )
+        return self.tokenizer.decode(output[0], skip_special_tokens=True)
+
+
+if __name__ == "__main__":
+    # Example Usage:
+    rag = RAGPipeline()
+    print(rag.generate_answer("Who is the President of NACOS?"))

requirements.txt

@@ -0,0 +1,12 @@
+gradio
+faiss-cpu
+sentence-transformers>=2.2.2
+scikit-learn>=1.1.3
+numpy>=1.24.3
+matplotlib>=3.7.1
+scipy>=1.11.1
+pandas>=2.0.1
+nlpaug>=1.1.10
+nltk>=3.8.1
+torch>=2.0.1
+transformers>=4.32.0