Spaces:
Running
Running
File size: 1,655 Bytes
5de4570 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
# %%
import os
import pandas as pd
import nest_asyncio
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).resolve().parents[1]))
from mythesis_chatbot import evaluation
from trulens.core import TruSession
from mythesis_chatbot.rag_setup import (
sentence_window_retrieval_setup,
)
import yaml
from trulens.dashboard.display import get_feedback_result
from trulens.dashboard import run_dashboard
# %%
with open(os.path.join("../../configs", "sentence_window.yaml"), "r") as f:
config = yaml.safe_load(f)
engine = sentence_window_retrieval_setup(
input_file="../../data/Master_Thesis.pdf", save_dir="../../data/indices", **config
)
# database_url=os.getenv("SUPABASE_CONNECTION_STRING")
tru = TruSession(database_url=os.getenv("SUPABASE_CONNECTION_STRING"))
tru.reset_database()
nest_asyncio.apply()
# %%
tru_recorder = evaluation.get_prebuilt_trulens_recorder(engine, config)
# %%
query = "Why?"
with tru_recorder as recording: # noqa: F841
response = engine.query(query) # noqa: F841
# %%
database = tru_recorder.db
# %%
rec = recording.get()
# get_feedback_result(rec, "Context Relevance")
for feedback, feedback_result in rec.wait_for_feedback_results().items():
print(feedback.name, feedback_result.result)
# database.insert_feedback(feedback_result)
# %%
evaluation.run_evals(
os.path.join("../../data/", "eval_questions.txt"), tru_recorder, engine
)
# %%
records, feedback = tru.get_records_and_feedback(app_ids=[])
records.head()
# %%
pd.set_option("display.max_colwidth", None)
records[["input", "output"] + feedback]
# %%
tru.get_leaderboard(app_ids=[])
# %%
tru.run_dashboard()
|