add files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/**/*.json filter=lfs diff=lfs merge=lfs -text
+data/**/* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,374 @@
+import os
+import json
+import numpy as np
+import streamlit as st
+from sentence_transformers import SentenceTransformer
+from openai import OpenAI
+import random
+import prompts
+
+
+# client = OpenAI(api_key=st.secrets["general"]["OpenAI_API"])
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+st.set_page_config(page_title="The AI Teaching Assistant")
+
+def generate_YT_file_names(base_path, embedding_model_name, chunk_tokens, overlap_tokens):
+    embedding_space_file_name = f'{base_path}/yt_embedding_space_{embedding_model_name}_tpc{chunk_tokens}_o{overlap_tokens}.json'
+    with open(embedding_space_file_name, 'r') as json_file:
+        loaded_data = json.load(json_file)
+    
+    embedding_space = np.array(loaded_data['embedding_space'])
+    return loaded_data['chunks'], embedding_space
+
+def generate_Latex_file_names(base_path, embedding_model_name, chunk_tokens, overlap_tokens):
+    embedding_space_file_name = f'{base_path}/latex_embedding_space_by_sections_{embedding_model_name}_tpc{chunk_tokens}_o{overlap_tokens}.json'
+    with open(embedding_space_file_name, 'r') as json_file:
+        loaded_data = json.load(json_file)
+    
+    embedding_space = np.array(loaded_data['embedding_space'])
+    return loaded_data['chunks'], embedding_space
+
+@st.cache_resource
+def load_text_data(json_file_name):
+    with open(json_file_name, 'r') as f:
+        data = json.load(f)
+    return data
+
+
+@st.cache_resource
+def load_embeddings(npy_file_name):
+    return np.load(npy_file_name)
+
+@st.cache_resource
+def load_model(model_name):
+    return SentenceTransformer(model_name)
+
+@st.cache_resource
+def load_summary(file_path):
+    with open(file_path, 'r') as file:
+        transcripts = json.load(file)
+    return transcripts
+
+
+def embed_question_openai(texts, model="text-embedding-3-small"):
+    response = client.embeddings.create(
+        input=texts,
+        model=model
+    )
+    return np.array(response.data[0].embedding)
+
+def embed_question(question, embedding_model):
+    if embedding_model == "text-embedding-3-small":
+        return embed_question_openai(question, embedding_model)
+    else:
+        return embedding_model.encode(question, convert_to_numpy=True)
+    
+def fixed_knn_retrieval(question_embedding, context_embeddings, top_k=5, min_k=1):
+    
+    # Normalize 
+    question_embedding = question_embedding / np.linalg.norm(question_embedding)
+    context_embeddings = context_embeddings / np.linalg.norm(context_embeddings, axis=1, keepdims=True)
+    
+    # Calculate cosine similarities between the question embedding and all context embeddings.
+    similarities = np.dot(context_embeddings, question_embedding)
+    # Sort the similarities in descending order and get the corresponding indices.
+    sorted_indices = np.argsort(similarities)[::-1]
+    # Select the top_k most similar contexts, ensuring at least min_k contexts are selected.
+    selected_indices = sorted_indices[:max(top_k, min_k)].tolist()
+    return selected_indices
+
+def sec_to_time(start_time):
+    return f"{start_time // 60:02}:{start_time % 60:02}"
+
+st.markdown("""
+    <style>
+    .video-wrapper {
+        position: relative;
+        padding-bottom: 56.25%;
+        height: 0;
+    }
+    .video-wrapper iframe {
+        position: absolute;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: 100%;
+    }
+    </style>
+    """, unsafe_allow_html=True)
+
+def get_youtube_embed(video_id, start_time=0, autoplay=0):
+    embed_code = f'''
+    <div class="video-wrapper">
+        <iframe src="https://www.youtube.com/embed/{video_id}?start={start_time}&autoplay={autoplay}&rel=0" 
+        frameborder="0" allowfullscreen></iframe>
+    </div>
+    '''
+    return embed_code
+
+
+disclaimer_contact =""":gray[AI Teaching Assistant is developed at the University of Southern California by Mostafa Faghih Shojaei, Rahul Gulati, Benjamin Jasperson, Shangshang Wang, Simone Cimolato, Dangli Cao, Willie Neiswanger, and Krishna Garikipati.]
+
+:gray[**Main Data Sources:**] [Introduction to Finite Element Methods (FEM) by Prof. Krishna Garikipati](https://www.youtube.com/playlist?list=PLJhG_d-Sp_JHKVRhfTgDqbic_4MHpltXZ) :gray[and] [The Finite Element Method: Linear Static and Dynamic Finite Element Analysis by Thomas J. R. Hughes](https://www.google.com/books/edition/_/cHH2n_qBK0IC?hl=en).
+
+:gray[**Disclaimer and Copyright Notice:**] :gray[1. AI-Generated Responses: Answers are generated using AI and, while thorough, may not always be 100% accurate. Please verify the information independently. 2. Content Ownership: All video content and lecture material referenced belong to their original creators. We encourage users to view the original material on verified platforms to ensure authenticity and accuracy. 3. Educational Fair Use: This tool is intended solely for educational purposes and operates under the principles of fair use. It is not authorized for commercial applications.]
+
+:gray[For any questions, concerns, or feedback about this application, please contact the development team directly.]
+"""
+
+# ---------------------------------------
+
+
+base_path = "data/"
+
+st.title(":red[AI Teaching Assistant]")
+# st.markdown("### Finite Element Methods")
+# st.markdown("### Based on Introduction to Finite Element Methods (FEM) by Prof. Krishna Garikipati")
+# st.markdown("##### [YouTube playlist of the FEM lectures](https://www.youtube.com/playlist?list=PLJhG_d-Sp_JHKVRhfTgDqbic_4MHpltXZ)")
+
+st.markdown(":gray[Welcome to] :red[AI Teaching Assistant]:gray[, developed at the] :red[University of Southern California]:gray[. This app leverages AI to provide expert answers to queries related to] :red[Finite Element Methods (FEM)]:gray[.]")
+
+# As the content is AI-generated, we strongly recommend independently verifying the information provided. 
+
+st.markdown(" ")
+st.markdown(" ")
+# st.divider()
+# Sidebar for settings
+with st.sidebar:
+    st.header("Settings")
+    # with st.container(border=True):
+    # Embedding model
+
+    model_name = st.selectbox("Choose content embedding model", [
+        "text-embedding-3-small",
+        # "text-embedding-3-large",
+        # "all-MiniLM-L6-v2", 
+        # "all-mpnet-base-v2"
+    ], 
+    # help="""
+    # Select the embedding model to use for encoding the retrieved text data. 
+    # Options include OpenAI's `text-embedding-3` models and two widely
+    # used SentenceTransformers models.
+    # """
+    )
+    
+    with st.container(border=True):
+        st.write('**Video lectures**')
+        yt_token_choice = st.select_slider("Token per content", [256, 512, 1024], value=256, help="Larger values lead to an increase in the length of each retrieved piece of content", key="yt_token_len")
+        yt_chunk_tokens = yt_token_choice
+        yt_max_content = {128: 32, 256: 16, 512: 8, 1024: 4}[yt_chunk_tokens]
+        top_k_YT = st.slider("Number of relevant content pieces to retrieve", 0, yt_max_content, 4, key="yt_token_num")
+        yt_overlap_tokens = yt_chunk_tokens // 4
+
+    # st.divider()
+    with st.container(border=True):
+        st.write('**Textbook**')
+        show_textbook = False
+        # show_textbook = st.toggle("Show Textbook Content", value=False)
+        latex_token_choice = st.select_slider("Token per content", [128, 256, 512, 1024], value=256, help="Larger values lead to an increase in the length of each retrieved piece of content", key="latex_token_len")
+        latex_chunk_tokens = latex_token_choice
+        latex_max_content = {128: 32, 256: 16, 512: 8, 1024: 4}[latex_chunk_tokens]
+        top_k_Latex = st.slider("Number of relevant content pieces to retrieve", 0, latex_max_content, 4, key="latex_token_num")
+        # latex_overlap_tokens = latex_chunk_tokens // 4
+        latex_overlap_tokens = 0
+
+    st.write(' ')
+    with st.expander('Expert model',expanded=False):
+        # st.write('**Expert model**')
+        # with st.container(border=True):
+        # Choose the LLM model
+
+        use_expert_answer = st.toggle("Use expert answer", value=True)
+        show_expert_responce = st.toggle("Show initial expert answer", value=False)
+
+        model = st.selectbox("Choose the LLM model", ["gpt-4o-mini", "gpt-3.5-turbo"], key='a1model')
+
+        # Temperature
+        expert_temperature = st.slider("Temperature", 0.0, 0.3, .2,  help="Defines the randomness in the next token prediction. Lower: More predictable and focused. Higher: More adventurous and diverse.", key='a1t')
+
+        expert_top_p = st.slider("Top P", 0.1, 0.3, 0.1, help="Defines the range of token choices the model can consider in the next prediction. Lower: More focused and restricted to high-probability options. Higher: More creative, allowing consideration of less likely options.", key='a1p')
+        
+
+    with st.expander('Synthesis model',expanded=False):
+
+        # with st.container(border=True):
+        # Choose the LLM model
+        model = st.selectbox("Choose the LLM model", ["gpt-4o-mini", "gpt-3.5-turbo"], key='a2model')
+
+        # Temperature
+        integration_temperature = st.slider("Temperature", 0.0, .3, .5, help="Defines the randomness in the next token prediction. Lower: More predictable and focused. Higher: More adventurous and diverse.", key='a2t')
+
+        integration_top_p = st.slider("Top P", 0.1, 0.5, .3, help="Defines the range of token choices the model can consider in the next prediction. Lower: More focused and restricted to high-probability options. Higher: More creative, allowing consideration of less likely options.", key='a2p')
+
+        
+
+# Main content area
+if "question" not in st.session_state:
+    st.session_state.question = ""
+
+def get_random_question():
+    with open(base_path + "/questions.txt", "r") as file:
+        questions = [line.strip() for line in file]
+    return random.choice(questions)
+
+text_area_placeholder = st.empty()
+question_help = "Including details or instructions improves the answer."
+st.session_state.question = text_area_placeholder.text_area(
+    "**Enter your question/query about Finite Element Method**",
+    height=120,
+    value=st.session_state.question,
+    help=question_help
+)
+
+_, col1, col2, _ = st.columns([4, 2, 4, 3]) 
+with col1:
+    submit_button_placeholder = st.empty()
+
+with col2:
+    if st.button("Random Question"):
+        while True:
+            random_question = get_random_question()
+            if random_question != st.session_state.question:
+                break
+        st.session_state.question = random_question
+        text_area_placeholder.text_area(
+            "**Enter your question:**",
+            height=120,
+            value=st.session_state.question,
+            help=question_help
+        )
+
+# Load YouTube and LaTeX data
+text_data_YT, context_embeddings_YT = generate_YT_file_names(base_path, model_name, yt_chunk_tokens, yt_overlap_tokens)
+text_data_Latex, context_embeddings_Latex = generate_Latex_file_names(base_path, model_name, latex_chunk_tokens, latex_overlap_tokens)
+
+summary = load_summary('data/KG_FEM_summary.json')
+
+if 'question_answered' not in st.session_state:
+    st.session_state.question_answered = False
+if 'context_by_video' not in st.session_state:
+    st.session_state.context_by_video = {}
+if 'context_by_section' not in st.session_state:
+    st.session_state.context_by_section = {}
+if 'answer' not in st.session_state:
+    st.session_state.answer = ""
+if 'playing_video_id' not in st.session_state:
+    st.session_state.playing_video_id = None
+
+if submit_button_placeholder.button("AI Answer", type="primary"): 
+    if st.session_state.question != "":
+        with st.spinner("Finding relevant contexts..."):
+            question_embedding = embed_question(st.session_state.question, model_name)
+            initial_max_k = int(0.1 * context_embeddings_YT.shape[0])
+            idx_YT = fixed_knn_retrieval(question_embedding, context_embeddings_YT, top_k=top_k_YT, min_k=0)
+            idx_Latex = fixed_knn_retrieval(question_embedding, context_embeddings_Latex, top_k=top_k_Latex, min_k=0)
+
+        with st.spinner("Answering the question..."):
+            relevant_contexts_YT = sorted([text_data_YT[i] for i in idx_YT], key=lambda x: x['order'])
+            relevant_contexts_Latex = sorted([text_data_Latex[i] for i in idx_Latex], key=lambda x: x['order'])
+
+            st.session_state.context_by_video = {}
+            for context_item in relevant_contexts_YT:
+                video_id = context_item['video_id']
+                if video_id not in st.session_state.context_by_video:
+                    st.session_state.context_by_video[video_id] = []
+                st.session_state.context_by_video[video_id].append(context_item)
+
+            st.session_state.context_by_section = {}
+            for context_item in relevant_contexts_Latex:
+                section_id = context_item['section']
+                if section_id not in st.session_state.context_by_section:
+                    st.session_state.context_by_section[section_id] = []
+                st.session_state.context_by_section[section_id].append(context_item)
+
+            context = ''
+            for i, (video_id, contexts) in enumerate(st.session_state.context_by_video.items(), start=1):
+                for context_item in contexts:
+                    start_time = int(context_item['start'])
+                    context += f'Video {i}, time: {sec_to_time(start_time)}:' + context_item['text'] + '\n\n'
+
+            for i, (section_id, contexts) in enumerate(st.session_state.context_by_section.items(), start=1):
+                context += f'Section {i} ({section_id}):\n'
+                for context_item in contexts:
+                    context += context_item['text'] + '\n\n'
+            
+            if use_expert_answer:
+                st.session_state.expert_answer = prompts.openai_domain_specific_answer_generation("Finite Element Method", st.session_state.question, model=model, temperature=expert_temperature, top_p=expert_top_p)
+            else:
+                st.session_state.expert_answer = 'No Expert Answer. Only use the context.'
+            answer = prompts.openai_context_integration("Finite Element Method", st.session_state.question, st.session_state.expert_answer, context, model=model, temperature=integration_temperature, top_p=integration_top_p)
+
+        if answer.split()[0] == "NOT_ENOUGH_INFO":
+            st.markdown("")
+            st.markdown("#### Query:")
+            st.markdown(prompts.fix_latex(st.session_state.question))
+            if show_expert_responce:
+                st.markdown("#### Initial Expert Answer:")
+                st.markdown(st.session_state.expert_answer)
+            st.markdown("#### Answer:")
+            st.write(":smiling_face_with_tear:")
+            st.markdown(answer.split('NOT_ENOUGH_INFO')[1])
+            st.divider()
+            st.caption(disclaimer_contact)
+            # st.caption("The AI Teaching Assistant project")
+            st.session_state.question_answered = False
+            st.stop()   
+        else:
+            st.session_state.answer = answer
+
+        st.session_state.question_answered = True
+
+    else:
+        st.markdown("")
+        st.write("Please enter a question. :smirk:")
+        st.session_state.question_answered = False
+
+if st.session_state.question_answered:
+    st.markdown("")
+    st.markdown("#### Query:")
+    st.markdown(prompts.fix_latex(st.session_state.question))
+    if show_expert_responce:
+        st.markdown("#### Initial Expert Answer:")
+        st.markdown(st.session_state.expert_answer)
+    st.markdown("#### Answer:")
+    st.markdown(st.session_state.answer)
+    
+    if top_k_YT > 0:
+        st.markdown("#### Retrieved content in lecture videos")
+        for i, (video_id, contexts) in enumerate(st.session_state.context_by_video.items(), start=1):
+            # with st.expander(f"**Video {i}** | {contexts[0]['title']}", expanded=True):
+            with st.container(border=True):
+                st.markdown(f"**Video {i} | {contexts[0]['title']}**")
+                video_placeholder = st.empty()
+                video_placeholder.markdown(get_youtube_embed(video_id, 0, 0), unsafe_allow_html=True)
+                st.markdown('')
+                with st.container(border=False):
+                    st.markdown("Retrieved Times")
+                    cols = st.columns([1 for i in range(len(contexts))] + [9 - len(contexts)])
+                    for j, context_item in enumerate(contexts):
+                        start_time = int(context_item['start'])
+                        label = sec_to_time(start_time)
+                        if cols[j].button(label, key=f"{video_id}_{start_time}"):
+                            if st.session_state.playing_video_id is not None:
+                                st.session_state.playing_video_id = None
+                                video_placeholder.empty()
+                            video_placeholder.markdown(get_youtube_embed(video_id, start_time, 1), unsafe_allow_html=True)
+                            st.session_state.playing_video_id = video_id
+                
+                with st.expander("Video Summary", expanded=False):
+                    # st.write("##### Video Overview:")
+                    st.markdown(summary[video_id])
+
+    if show_textbook and top_k_Latex > 0:
+        st.markdown("#### Retrieved content in textbook",help="The Finite Element Method: Linear Static and Dynamic Finite Element Analysis")
+        for i, (section_id, contexts) in enumerate(st.session_state.context_by_section.items(), start=1):
+            # with st.expander(f"**Section {i} | {section_id}**", expanded=True):
+            st.markdown(f"**Section {i} | {section_id}**")
+            for context_item in contexts:
+                st.markdown(context_item['text'])
+                st.divider()
+
+st.markdown(" ")
+st.divider()
+st.caption(disclaimer_contact)

data/KG_FEM_summary.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9179743956afab296c02eec5b6b82f8bc49a449721c46ddd9634d7c4be4053a
+size 203883

data/questions.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d23dbdf124c14dfb1c2224708711b4c892b5874f800e7e06240261f229468d61
+size 456242

data/yt_embedding_space_text-embedding-3-small_tpc1024_o256.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0bb1fc517dfa8eeea1f7545bec577556e7bd170ec885cdf25eab3f5d665d2ba
+size 17109772

data/yt_embedding_space_text-embedding-3-small_tpc256_o64.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f69373b4df1d9bef5a64a68d898c596fee6007c26a7ac6ff58d56f786c93d60
+size 62427532

data/yt_embedding_space_text-embedding-3-small_tpc512_o128.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1cc71a26650538650196102342e9f48242eedf5ed2bbf1fb4c2299691edd6c6
+size 31637503

prompts.py

@@ -0,0 +1,119 @@
+from openai import OpenAI
+import re
+import os
+
+
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+def fix_latex(text):
+    text = re.sub(r"\\\(", r"$",text)
+    text = re.sub(r"\\\)", r"$",text)
+    text = re.sub(r"\\\[", r"$$",text)
+    text = re.sub(r"\\\]", r"$$",text)
+    return text
+
+
+# Step 1: Domain-Specific Answer Generation
+def openai_domain_specific_answer_generation(subject, question, model="gpt4o-mini", temperature=0.3, top_p=0.1):
+    system_prompt = f"""
+    You are a highly specialized assistant for the subject {subject}. Provide a direct and focused answer to the following question based on your specialized training.
+    """
+    
+    prompt = f"""
+    Question:
+    {question}
+
+    Answer (provide a precise, domain-specific response):
+    """
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": system_prompt
+            },
+            {
+                "role": "user",
+                "content": prompt
+            }
+        ],
+        temperature=temperature,  # Set low for deterministic and precise responses.
+        top_p=top_p,  # Focus on high-probability outputs to ensure accuracy.
+        frequency_penalty=0.1,  # Reduce repetition of technical terms.
+        presence_penalty=0.0  # Prevent introduction of unrelated ideas.
+    )
+    return fix_latex(response.choices[0].message.content)
+
+
+
+# Step 2: Context Integration
+def openai_context_integration(subject_matter, query, expert_answer, retrieved_context, model="gpt4o-mini", temperature=0.3, top_p=0.3):
+    system_prompt = f"""
+    You are an AI teaching assistant for a {subject_matter} course. Your task is to answer questions based EXCLUSIVELY on the content provided from the professor's teaching materials. Do NOT use any external knowledge or information not present in the given context.
+
+    IMPORTANT: Before proceeding, carefully analyze the provided context and the question. If the context lacks sufficient information to answer the question adequately, respond EXACTLY as follows and then STOP:
+    \"NOT_ENOUGH_INFO The provided context doesn't contain enough information to fully answer this question. You may want to increase the number of relevant context passages or adjust the options and try again.\"
+
+    If the context is sufficient, continue with the remaining guidelines.
+
+    Guidelines:
+    1. Strictly adhere to the information in the context. Do not make assumptions or use general knowledge outside of the provided materials.
+
+    2. For partial answers:
+       a) Provide the information you can based on the context.
+       b) Clearly identify which aspects of the question you cannot address due to limited context.
+
+    3. Referencing:
+       a) Always cite your sources by referencing the video number and the given time in brackets and **bold** (e.g., [**Video 3, time 03:14**]) after each piece of information you use in your answer.
+       b) You may cite multiple references if they discuss the same content (e.g., [**Video 3, time 03:14; Video 1, time 12:04**]). However, try to reference them separately if they cover different aspects of the answer.
+
+    4. Length of response:
+       a) Use approximately 120-200 tokens for each video referenced.
+       b) If referencing multiple videos that discuss the same content, you can use a combined total of 120-200 tokens for all refrences.
+
+    5. Style and Formatting:
+        a) Provide the answer in markdown format.
+        b) Do not use any titles, sections, or subsections. Use mainly paragraphs. Bold text, items, and bullet points if it helps.
+        c) Symbols and equations within the text MUST be placed between $ and $, e.g., $x=0$ is the min of $\sigma(x)=x^2$.
+        d) For equations between paragraphs, use \n\n$ and $\n\n. For example, in the following equation: \n\n$ E = mc^2 $\n\n, note $c$ as the speed of light.
+
+    6. If multiple interpretations of the question are possible based on the context, acknowledge this and provide answers for each interpretation.
+
+    7. Use technical language appropriate for a {subject_matter} course, but be prepared to explain complex terms if asked.
+
+    8. If the question involves calculations, show your work step-by-step, citing the relevant formulas or methods from the context.
+    """
+
+    prompt = f"""
+    Question:
+    {query}
+
+    Direct Answer:
+    {expert_answer}
+
+    Retrieved Context:
+    {retrieved_context}
+
+    Final Answer:
+
+    """
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": system_prompt
+            },
+            {
+                "role": "user",
+                "content": prompt
+            }
+        ],
+        temperature=temperature,  # Maintain some flexibility for smooth blending.
+        top_p=top_p,  # Prioritize high-probability outputs to stay focused on the inputs.
+        frequency_penalty=0.1,  # Allow necessary repetition for clarity.
+        presence_penalty=0.0  # Neutral to avoid introducing unrelated ideas.
+    )
+    return fix_latex(response.choices[0].message.content)

requirements.txt

@@ -1,5 +1,5 @@
 numpy==1.26.3
 openai==1.57.0
 sentence-transformers==2.7.0
-streamlit==1.40.2
-python-dotenv==1.0.1
+streamlit==1.43.2
+python-dotenv==1.0.1