Fresh start

app.py

@@ -1,35 +0,0 @@
-# app.py
-
-import os
-from flask import Flask, jsonify
-from utils.data_loader import download_dataset, save_metadata
-from utils.vector_utils import create_vector_db
-from config import v_auth_token, v_vector_folder, v_metadata_file
-
-app = Flask(__name__)
-
-@app.route('/health', methods=['GET'])
-def health_check():
-    return jsonify({"status": "healthy"}), 200
-
-@app.route('/initialize', methods=['POST'])
-def initialize():
-    if not v_auth_token:
-        return jsonify({"error": "Authentication token not found"}), 500
-
-    # Ensure writable base directory and subdirectories are initialized
-    os.makedirs(v_vector_folder, exist_ok=True)
-    if not os.path.exists(v_metadata_file):
-        save_metadata(v_metadata_file, {})
-
-    print("Starting Vector Database Creation...")
-
-    v_dataset_path = download_dataset(v_auth_token, v_metadata_file, v_vector_folder)
-    if not v_dataset_path:
-        return jsonify({"message": "Vector database is up-to-date"}), 200
-    else:
-        create_vector_db(v_dataset_path, v_vector_folder, v_auth_token)
-        return jsonify({"message": "Vector database created successfully"}), 200
-
-if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860)

config.py

@@ -1,14 +0,0 @@
-# config.py
-
-import os
-
-# Dataset configuration
-v_dataset_url = "https://huggingface.co/datasets/vishalsh13/Dataset1/tree/main"
-
-# Authentication token retrieved from Hugging Face secret
-v_auth_token = os.getenv("hkey")  # The secret name is `hkey`
-
-# Paths for vector database and metadata
-v_base_path = "/tmp/vector_db"  # Writable directory
-v_vector_folder = os.path.join(v_base_path, "vectors")
-v_metadata_file = os.path.join(v_base_path, "metadata.json")

utils/data_loader.py

@@ -1,42 +0,0 @@
-# utils/data_loader.py
-
-import os
-import json
-from huggingface_hub import hf_hub_download
-
-def download_dataset(v_auth_token, v_metadata_file, v_vector_folder):
-    v_current_metadata = fetch_metadata()
-    v_existing_metadata = load_metadata(v_metadata_file)
-
-    if v_current_metadata == v_existing_metadata:
-        print("No updates detected. Skipping vector creation.")
-        return False
-
-    # Define the specific file to download
-    v_filename = "train.csv"  # Replace this with the actual filename in your repository
-    v_dataset_path = hf_hub_download(
-        repo_id="vishalsh13/Dataset1",
-        repo_type="dataset",
-        #subfolder="data",  # Adjust or remove subfolder as needed
-        filename=v_filename,
-        token=v_auth_token
-    )
-
-    print("Dataset downloaded successfully.")
-    save_metadata(v_metadata_file, v_current_metadata)
-    return v_dataset_path
-
-def fetch_metadata():
-    # Simulate fetching metadata (e.g., hash or timestamp)
-    return {"dataset_version": "v1.0"}
-
-def load_metadata(v_metadata_file):
-    if os.path.exists(v_metadata_file):
-        with open(v_metadata_file, "r") as file:
-            return json.load(file)
-    return {}
-
-def save_metadata(v_metadata_file, v_metadata):
-    os.makedirs(os.path.dirname(v_metadata_file), exist_ok=True)
-    with open(v_metadata_file, "w") as file:
-        json.dump(v_metadata, file)

utils/logger.py

@@ -1,11 +0,0 @@
-# utils/logger.py
-
-import logging
-
-def setup_logger(v_log_file="app.log"):
-    logging.basicConfig(
-        filename=v_log_file,
-        level=logging.INFO,
-        format="%(asctime)s - %(levelname)s - %(message)s"
-    )
-    return logging.getLogger()

utils/vector_utils.py

@@ -1,52 +0,0 @@
-# utils/vector_utils.py
-
-import os
-import faiss
-import numpy as np
-from huggingface_hub import Repository
-from sentence_transformers import SentenceTransformer
-
-def create_vector_db(v_dataset_path, v_vector_folder, v_auth_token):
-    # Initialize the model
-    obj_model = SentenceTransformer('all-MiniLM-L6-v2')
-
-    # Load and process data
-    with open(v_dataset_path, 'r') as file:
-        v_data = file.readlines()
-
-    v_embeddings = obj_model.encode(v_data)
-
-    # Save vectors locally
-    os.makedirs(v_vector_folder, exist_ok=True)
-    v_vector_file = os.path.join(v_vector_folder, "vector_index")
-    v_index = faiss.IndexFlatL2(v_embeddings.shape[1])
-    v_index.add(np.array(v_embeddings))
-    faiss.write_index(v_index, v_vector_file)
-
-    print(f"Vector database created and saved locally at {v_vector_file}")
-
-    # Save vector file back to Hugging Face dataset repository
-    upload_to_huggingface_repo(v_vector_file, v_auth_token)
-    print("Vector file successfully uploaded to Hugging Face dataset repository.")
-
-def upload_to_huggingface_repo(v_file_path, v_auth_token):
-    """
-    Upload the given file to the Hugging Face dataset repository.
-    """
-    v_repo_id = "vishalsh13/Dataset1"  # Replace with your repository name
-    v_repo = Repository(
-        local_dir="temp_repo",
-        clone_from=v_repo_id,
-        use_auth_token=v_auth_token
-    )
-
-    # Copy the file to the repository directory
-    os.makedirs(v_repo.local_dir, exist_ok=True)
-    v_dest_path = os.path.join(v_repo.local_dir, os.path.basename(v_file_path))
-    os.replace(v_file_path, v_dest_path)
-
-    # Commit and push the changes
-    v_repo.git_add(v_dest_path)
-    v_repo.git_commit("Upload updated vector file.")
-    v_repo.git_push()
-    print(f"Uploaded {os.path.basename(v_file_path)} to Hugging Face repository: {v_repo_id}")