updated source code

src/mythesis_chatbot/evaluation.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+import numpy as np
+from tqdm import tqdm
+from trulens.apps.llamaindex import TruLlama
+from trulens.core import Feedback
+from trulens.providers.openai import OpenAI
+
+from mythesis_chatbot.utils import get_config_hash
+
+
+def run_evals(eval_questions_path: Path, tru_recorder, query_engine):
+
+    eval_questions = []
+    with open(eval_questions_path) as file:
+        for line in file:
+            item = line.strip()
+            eval_questions.append(item)
+
+    for question in tqdm(eval_questions):
+        with tru_recorder as recording:  # noqa: F841
+            response = query_engine.query(question)  # noqa: F841
+
+
+# Feedback function
+def f_answer_relevance(provider=OpenAI(), name="Answer Relevance"):
+    return Feedback(provider.relevance_with_cot_reasons, name=name).on_input_output()
+
+
+# Feedback function
+def f_context_relevance(
+    provider=OpenAI(),
+    context=TruLlama.select_source_nodes().node.text,
+    name="Context Relevance",
+):
+    return (
+        Feedback(provider.relevance, name=name)
+        .on_input()
+        .on(context)
+        .aggregate(np.mean)
+    )
+
+
+# Feedback function
+def f_groundedness(
+    provider=OpenAI(),
+    context=TruLlama.select_source_nodes().node.text,
+    name="Groundedness",
+):
+    return (
+        Feedback(
+            provider.groundedness_measure_with_cot_reasons,
+            name=name,
+        )
+        .on(context)
+        .on_output()
+    )
+
+
+def get_prebuilt_trulens_recorder(
+    query_engine, query_engine_config: dict[str, str | int]
+):
+    app_name = query_engine_config["rag_mode"]
+    app_version = get_config_hash(query_engine_config)
+
+    tru_recorder = TruLlama(
+        query_engine,
+        app_name=app_name,
+        app_version=app_version,
+        metadata=query_engine_config,
+        feedbacks=[f_answer_relevance(), f_context_relevance(), f_groundedness()],
+    )
+    return tru_recorder

src/mythesis_chatbot/rag_setup.py

@@ -25,9 +25,11 @@ from llama_index.core.retrievers import AutoMergingRetriever
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.llms.openai import OpenAI
 
-from src.mythesis_chatbot.utils import get_config_hash, get_openai_api_key
+from mythesis_chatbot.utils import get_config_hash, get_openai_api_key
 
-SupportedRags = Literal["basic", "sentence window retrieval", "auto-merging retrieval"]
+SupportedRags = Literal[
+    "classic retrieval", "sentence window retrieval", "auto-merging retrieval"
+]
 SupportedOpenAIllms = Literal["gpt-4o-mini", "gpt-3.5-turbo"]
 SupportedEmbedModels = Literal["BAAI/bge-small-en-v1.5"]
 SupportedRerankModels = Literal["cross-encoder/ms-marco-MiniLM-L-2-v2"]
@@ -167,6 +169,7 @@ def sentence_window_retrieval_setup(
     similarity_top_k: int = 6,
     rerank_model: SupportedRerankModels = "cross-encoder/ms-marco-MiniLM-L-2-v2",
     rerank_top_n: int = 2,
+    **kwargs
 ):
 
     openai.api_key = get_openai_api_key()
@@ -204,6 +207,7 @@ def automerging_retrieval_setup(
     similarity_top_k: int = 6,
     rerank_model: SupportedRerankModels = "cross-encoder/ms-marco-MiniLM-L-2-v2",
     rerank_top_n: int = 2,
+    **kwargs
 ):
     openai.api_key = get_openai_api_key()
 
@@ -239,6 +243,7 @@ def basic_rag_setup(
     similarity_top_k: int = 6,
     rerank_model: SupportedRerankModels = "cross-encoder/ms-marco-MiniLM-L-2-v2",
     rerank_top_n: int = 2,
+    **kwargs
 ):
     openai.api_key = get_openai_api_key()
 

src/mythesis_chatbot/run_evaluation.py

@@ -0,0 +1,71 @@
+# %%
+import os
+import pandas as pd
+import nest_asyncio
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+from mythesis_chatbot import evaluation
+from trulens.core import TruSession
+from mythesis_chatbot.rag_setup import (
+    sentence_window_retrieval_setup,
+)
+import yaml
+from trulens.dashboard.display import get_feedback_result
+from trulens.dashboard import run_dashboard
+
+# %%
+
+with open(os.path.join("../../configs", "sentence_window.yaml"), "r") as f:
+    config = yaml.safe_load(f)
+
+engine = sentence_window_retrieval_setup(
+    input_file="../../data/Master_Thesis.pdf", save_dir="../../data/indices", **config
+)
+
+# database_url=os.getenv("SUPABASE_CONNECTION_STRING")
+tru = TruSession(database_url=os.getenv("SUPABASE_CONNECTION_STRING"))
+tru.reset_database()
+nest_asyncio.apply()
+
+
+# %%
+
+tru_recorder = evaluation.get_prebuilt_trulens_recorder(engine, config)
+# %%
+
+query = "Why?"
+with tru_recorder as recording:  # noqa: F841
+    response = engine.query(query)  # noqa: F841
+
+# %%
+database = tru_recorder.db
+
+# %%
+
+rec = recording.get()
+# get_feedback_result(rec, "Context Relevance")
+
+for feedback, feedback_result in rec.wait_for_feedback_results().items():
+    print(feedback.name, feedback_result.result)
+    # database.insert_feedback(feedback_result)
+
+
+# %%
+evaluation.run_evals(
+    os.path.join("../../data/", "eval_questions.txt"), tru_recorder, engine
+)
+
+# %%
+records, feedback = tru.get_records_and_feedback(app_ids=[])
+records.head()
+# %%
+pd.set_option("display.max_colwidth", None)
+records[["input", "output"] + feedback]
+# %%
+
+tru.get_leaderboard(app_ids=[])
+# %%
+tru.run_dashboard()