backend/semantic_search.py

import lancedb
import os
import gradio as gr
from sentence_transformers import SentenceTransformer
from pathlib import Path
from dotenv import load_dotenv
import logging

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Load environment variables from the .env file
load_dotenv()

# Determine the LanceDB path and log it
current_working_dir = Path(os.getcwd())
db_path = current_working_dir / ".lancedb"
logger.info(f"Database path: {db_path}")

# List contents of the LanceDB directory
if db_path.exists():
    lancedb_contents = os.listdir(db_path)
    logger.info(f"Contents of the LanceDB directory: {lancedb_contents}")
else:
    logger.error(f"LanceDB directory does not exist at path: {db_path}")

db = lancedb.connect(db_path)

# List and log all tables in the database
table_names = db.table_names()
logger.info(f"Available LanceDB Tables: {table_names}")

model1_fixed_path = db_path / "model1_fixed.lance"
if model1_fixed_path.exists():
    model1_fixed_contents = os.listdir(model1_fixed_path)
    logger.info(f"Contents of the model1_fixed.lance folder: {model1_fixed_contents}")
else:
    logger.error(f"model1_fixed.lance directory does not exist at path: {model1_fixed_path}")

MODEL1_STRATEGY1 = "model1_fixed"
MODEL2_STRATEGY1 = "model2_fixed"
MODEL3_STRATEGY1 = "model3_fixed"

VECTOR_COLUMN = os.getenv("VECTOR_COLUMN", "vector")
TEXT_COLUMN = os.getenv("TEXT_COLUMN", "text")
BATCH_SIZE = int(os.getenv("BATCH_SIZE", 32))

retriever = SentenceTransformer(os.getenv("EMB_MODEL"))

def get_table_name():
    emb_model = os.getenv("EMB_MODEL")
    if emb_model == "sentence-transformers/all-MiniLM-L6-v2":
        return MODEL1_STRATEGY1
    elif emb_model == "BAAI/bge-large-en-v1.5":
        return MODEL2_STRATEGY1
    elif emb_model == "openai/text-embedding-ada-002":
        return MODEL3_STRATEGY1
    else:
        raise ValueError(f"Unsupported embedding model: {emb_model}")

def retrieve(query, k):
    table_name = get_table_name()
    TABLE = db.open_table(table_name)
    query_vec = retriever.encode(query)
    try:
        documents = TABLE.search(query_vec, vector_column_name=VECTOR_COLUMN).limit(k).to_list()
        documents = [doc[TEXT_COLUMN] for doc in documents]

        return documents

    except Exception as e:
        raise gr.Error(str(e))
    
if __name__ == "__main__":
    res = retrieve("What is transformer?", 4)
    print(res)