Update app.py

app.py

@@ -1,85 +1,85 @@
-import gradio as gr
 import pandas as pd
-import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
-import openai
+from sentence_transformers import SentenceTransformer, util
+from transformers import pipeline
+import torch
+import gradio as gr
 import os
 
-# Set up your OpenAI API key (consider using environment variables for security)
-openai.api_key = os.getenv("OPENAI_API_KEY")
-
-# Load course data from CSV file
-df = pd.read_csv('course_data.csv')
-
-# Combine relevant columns into one text representation for each course
-def combine_course_text(row):
-    return f"{row['Course Title']} {row['Description']} {row['All Chapter Titles']} {row['All Lessons']}"
-
-df['combined_text'] = df.apply(combine_course_text, axis=1)
-
-# Function to get embeddings for text using OpenAI's API
-def get_embedding(text):
-    try:
-        response = openai.Embedding.create(
-            input=text,
-            model="text-embedding-ada-002"
-        )
-        embedding = response['data'][0]['embedding']
-        return embedding
-    except Exception as e:
-        print(f"Error while getting embedding: {e}")
-        return None
-
-# Pre-compute embeddings for all courses in the dataset
-course_embeddings = []
-course_titles = df['Course Title'].tolist()
-
-for text in df['combined_text']:
-    embedding = get_embedding(text)
-
-    if embedding is not None:
-        course_embeddings.append(embedding)
-
-    else:
-        print(f"Failed to generate embedding for: {text}")
-
-# Convert embeddings to numpy array (for cosine_similarity to work properly)
-course_embeddings = np.array(course_embeddings)
-
-# Function to search courses based on a query
-def search_courses(query):
-    # Get embedding for query
-    query_embedding = get_embedding(query)
-
-    if query_embedding is None:
-        return "Error in generating query embedding."
-
-    # Compute cosine similarity between query embedding and course embeddings
-    similarities = cosine_similarity([query_embedding], course_embeddings)
-
-    # Sort by similarity
-    sorted_indices = np.argsort(similarities[0])[::-1]
-
-    # Get top 3 courses based on similarity
-    top_courses = [course_titles[i] for i in sorted_indices[:3]]
-
-    return top_courses
-
-# Gradio Interface
-def gradio_search(query):
-    if query.strip():
-        results = search_courses(query)
-        if len(results) < 3:
-            results.extend(["No results"] * (3 - len(results)))
-        return "Top relevant courses:\n1. " + results[0] + "\n2. " + results[1] + "\n3. " + results[2]
-
-# Create Gradio interface
-interface = gr.Interface(
-    fn=gradio_search,
-    inputs="text",
-    outputs="text",
-    title="Smart Course Search",
-    description="Enter a query and get the most relevant courses from the dataset.")
-
-# Launch the Gradio interface
-interface.launch(share=True)
+# Load the dataset
+csv_file_path = os.path.join(os.getcwd(), 'Analytics_Vidhya_Free_Course_data.csv')
+df = pd.read_csv(csv_file_path, encoding='Windows-1252')
+df.fillna('', inplace=True)
+
+# Load the pre-trained model for embeddings
+model = SentenceTransformer('multi-qa-mpnet-base-dot-v1')
+
+# Combine title and description to create a full text for each course
+df['full_text'] = df.iloc[:, 0] + " " + df.iloc[:, 1] + " " + df['Instructor Name'] + " " + df['Rating'].astype(str) + " " + df['Category']
+
+# Convert full course texts into embeddings
+course_embeddings = model.encode(df['full_text'].tolist(), convert_to_tensor=True)
+
+# Load a model for text generation (e.g., BART)
+generator = pipeline('text2text-generation', model='facebook/bart-large-cnn')
+
+def expand_query(query):
+    paraphraser = pipeline('text2text-generation', model='Vamsi/T5_Paraphrase_Paws')
+    expanded_queries = paraphraser(query, num_return_sequences=3, max_length=50, do_sample=True)
+    return [q['generated_text'] for q in expanded_queries]
+
+def generate_description(query):
+    response = generator(query, max_length=100, num_return_sequences=1)
+    return response[0]['generated_text']
+
+def search_courses(query, level_filter=None, category_filter=None, top_k=3):
+    expanded_queries = expand_query(query)
+    all_similarities = []
+
+    for expanded_query in expanded_queries:
+        query_embedding = model.encode(expanded_query, convert_to_tensor=True)
+        similarities = util.pytorch_cos_sim(query_embedding, course_embeddings)[0]
+        all_similarities.append(similarities)
+
+    aggregated_similarities = torch.max(torch.stack(all_similarities), dim=0)[0]
+    filtered_df = df.copy()
+
+    if level_filter and level_filter != "Nil":
+        filtered_df = filtered_df[filtered_df['Level of Difficulty'] == level_filter]
+    if category_filter and category_filter != "NIL":
+        filtered_df = filtered_df[filtered_df['Category'] == category_filter]
+
+    if filtered_df.empty:
+        return "<p>No matching courses found.</p>"
+
+    filtered_similarities = aggregated_similarities[filtered_df.index]
+    top_results = filtered_similarities.topk(k=min(top_k, len(filtered_similarities)))
+
+    results = []
+    for idx in top_results.indices:
+        idx = int(idx)
+        course_title = filtered_df.iloc[idx]['Course Title']
+        course_description = filtered_df.iloc[idx, 1]
+        course_url = filtered_df.iloc[idx, -1]
+        generated_description = generate_description(course_title + " " + course_description)
+        course_link = f'<a href="{course_url}" target="_blank">{course_title}</a>'
+        results.append(f"<strong>{course_link}</strong><br>{course_description}<br>{generated_description}<br><br>")
+
+    return "<ol>" + "".join([f"<li>{result}</li>" for result in results]) + "</ol>"
+
+def create_gradio_interface():
+    with gr.Blocks() as demo:
+        gr.Markdown("#  Analytics Vidhya Free Courses")
+        gr.Markdown("Enter your query and use filters to narrow down the search.")
+        query = gr.Textbox(label=" Search for a course", placeholder="Enter course topic or description")
+        with gr.Accordion(" Filters", open=False):
+            level_filter = gr.Dropdown(choices=["Beginner", "Intermediate", "Advanced", "Nil"], label=" Course Level", multiselect=False)
+            category_filter = gr.Dropdown(choices=["Data Science", "Machine Learning", "Deep Learning", "AI", "NLP", "NIL"], label=" Category", multiselect=False)
+        search_button = gr.Button("Search")
+        output = gr.HTML(label="Search Results")
+        search_button.click(fn=search_courses, inputs=[query, level_filter, category_filter], outputs=output)
+
+    return demo
+
+# Launch Gradio interface
+demo = create_gradio_interface()
+demo.launch()