Delete retrieval

retrieval/hybrid_search.py

@@ -1,49 +0,0 @@
-# src/retrieval/hybrid_search.py
-
-import numpy as np
-from rank_bm25 import BM25Okapi
-from typing import List, Dict, Any
-import pandas as pd
-from src.embeddings.embedder import Embedder
-from src.retrieval.vector_store import VectorStore
-
-class HybridRetriever:
-    def __init__(self, df: pd.DataFrame, vector_store: VectorStore, embedder: Embedder, alpha: float = 0.5):
-        self.df = df
-        self.vector_store = vector_store
-        self.embedder = embedder
-        self.alpha = alpha
-        tokenized_corpus = [doc.lower().split() for doc in df['description']]
-        self.bm25 = BM25Okapi(tokenized_corpus)
-    
-    def retrieve(self, query: str, filtered_df: pd.DataFrame, top_k: int = 3) -> List[Dict[str, Any]]:
-        filtered_indices = filtered_df.index.tolist()
-        filtered_texts = filtered_df['description'].tolist()
-        filtered_ids = [str(row['id']) for _, row in filtered_df.iterrows()]
-        
-        if not filtered_texts:
-            return []
-        
-        query_embedding = self.embedder.embed([query])[0]
-        dense_results = self.vector_store.query(query_embedding, top_k=top_k * 2)
-        dense_ids = [id for id in dense_results['ids'][0] if id in filtered_ids]
-        dense_scores = [1 - dist for dist, id in zip(dense_results['distances'][0], dense_results['ids'][0]) if id in filtered_ids]
-        
-        tokenized_query = query.lower().split()
-        bm25_scores = self.bm25.get_scores(tokenized_query)
-        bm25_scores_filtered = [bm25_scores[i] for i in filtered_indices]
-        bm25_top_k = np.argsort(bm25_scores_filtered)[::-1][:top_k * 2]
-        bm25_ids = [filtered_ids[i] for i in bm25_top_k]
-        bm25_scores = [bm25_scores_filtered[i] for i in bm25_top_k]
-        
-        dense_scores = np.array(dense_scores) / np.max(dense_scores) if dense_scores else dense_scores
-        bm25_scores = np.array(bm25_scores) / np.max(bm25_scores) if bm25_scores else bm25_scores
-        
-        combined_scores = {}
-        for idx, dense_id in enumerate(dense_ids):
-            combined_scores[int(dense_id)] = combined_scores.get(int(dense_id), 0) + self.alpha * dense_scores[idx]
-        for idx, bm25_id in enumerate(bm25_ids):
-            combined_scores[int(bm25_id)] = combined_scores.get(int(bm25_id), 0) + (1 - self.alpha) * bm25_scores[idx]
-        
-        sorted_ids = sorted(combined_scores, key=combined_scores.get, reverse=True)[:top_k]
-        return [self.df[self.df['id'] == id].iloc[0].to_dict() for id in sorted_ids]

retrieval/keyword_filter.py

@@ -1,41 +0,0 @@
-# src/retrieval/keyword_filter.py
-
-import pandas as pd
-from typing import Dict, Any
-
-def filter_restaurants(df: pd.DataFrame, parsed_query: Dict[str, Any]) -> pd.DataFrame:
-    """
-    Filter restaurants based on extracted features from the query.
-    
-    Args:
-        df (pd.DataFrame): DataFrame containing restaurant data.
-        parsed_query (Dict[str, Any]): Parsed query with features.
-    
-    Returns:
-        pd.DataFrame: Filtered DataFrame.
-    """
-    filtered_df = df.copy()
-    
-    if parsed_query.get("cuisine"):
-        filtered_df = filtered_df[filtered_df["cuisine"].str.lower() == parsed_query["cuisine"].lower()]
-    
-    if parsed_query.get("menu"):
-        filtered_df = filtered_df[filtered_df["dishes"].apply(
-            lambda dishes: any(item.lower() in [d.lower() for d in dishes] for item in parsed_query["menu"])
-        )]
-    
-    if parsed_query.get("price_range"):
-        filtered_df = filtered_df[filtered_df["price_range"].str.lower() == parsed_query["price_range"].lower()]
-    
-    distance = parsed_query.get("distance")
-    if isinstance(distance, (int, float)):
-        filtered_df = filtered_df[filtered_df["distance"] <= distance]
-    elif distance in ["nearby", "close"]:
-        filtered_df = filtered_df[filtered_df["distance"] <= 2.0]
-    elif distance == "far":
-        filtered_df = filtered_df[filtered_df["distance"] <= 10.0]
-    
-    if parsed_query.get("rating"):
-        filtered_df = filtered_df[filtered_df["rating"] >= parsed_query["rating"]]
-    
-    return filtered_df

retrieval/vector_store.py

@@ -1,34 +0,0 @@
-# src/retrieval/vector_store.py
-
-from langchain_community.vectorstores import Chroma
-from langchain_core.documents import Document
-import numpy as np
-from typing import List, Dict, Any
-
-class VectorStore:
-    def __init__(self, embedding_function):
-        self.embedding_function = embedding_function
-        self.collection = None
-    
-    def add_documents(self, documents: List[str], embeddings: List[np.ndarray], ids: List[str]):
-        langchain_docs = [Document(page_content=doc, metadata={"id": id}) for doc, id in zip(documents, ids)]
-        self.collection = Chroma.from_documents(
-            documents=langchain_docs,
-            embedding=self.embedding_function,
-            ids=ids,
-            persist_directory="./chroma_db"
-        )
-        self.collection.persist()
-    
-    def query(self, query_embedding: np.ndarray, top_k: int = 5) -> Dict[str, Any]:
-        results = self.collection.similarity_search_by_vector(
-            embedding=query_embedding,
-            k=top_k
-        )
-        ids = [doc.metadata["id"] for doc in results]
-        distances = [1 - np.dot(query_embedding, doc.vector) / (np.linalg.norm(query_embedding) * np.linalg.norm(doc.vector)) 
-                     if hasattr(doc, "vector") else 1.0 for doc in results]
-        return {
-            "ids": [ids],
-            "distances": [distances]
-        }