app.py

import os
import pandas as pd
import torch
from sentence_transformers import SentenceTransformer, util
import faiss
import numpy as np
import pickle
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import scipy.special
from sklearn.feature_extraction.text import TfidfVectorizer
from flask import Flask, request, jsonify
import logging
from pymongo import MongoClient

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Disable tokenizers parallelism to avoid fork-related deadlocks
os.environ["TOKENIZERS_PARALLELISM"] = "false"

# MongoDB connection
MONGO_URI = "mongodb://muhammadbinimran1001:[email protected]:27017,dsm-shard-00-01.inrzs.mongodb.net:27017,dsm-shard-00-02.inrzs.mongodb.net:27017/?ssl=true&replicaSet=atlas-nbg4er-shard-0&authSource=admin&retryWrites=true&w=majority"
client = MongoClient(MONGO_URI)
db = client.get_database("test")
users_collection = db["users"]
courses_collection = db["courses"]
jobs_collection = db["jobs"]

# Paths for saving artifacts
MODEL_DIR = "./saved_models"
FALLBACK_MODEL_DIR = "/tmp/saved_models"

try:
    os.makedirs(MODEL_DIR, exist_ok=True)
    logger.info(f"Using model directory: {MODEL_DIR}")
    chosen_model_dir = MODEL_DIR
except Exception as e:
    logger.warning(f"Failed to create {MODEL_DIR}: {e}. Using fallback directory.")
    os.makedirs(FALLBACK_MODEL_DIR, exist_ok=True)
    chosen_model_dir = FALLBACK_MODEL_DIR

# Update paths
UNIVERSAL_MODEL_PATH = os.path.join(chosen_model_dir, "universal_model")
DETECTOR_MODEL_PATH = os.path.join(chosen_model_dir, "detector_model")
TFIDF_PATH = os.path.join(chosen_model_dir, "tfidf_vectorizer.pkl")
SKILL_TFIDF_PATH = os.path.join(chosen_model_dir, "skill_tfidf.pkl")
QUESTION_ANSWER_PATH = os.path.join(chosen_model_dir, "question_to_answer.pkl")
FAISS_INDEX_PATH = os.path.join(chosen_model_dir, "faiss_index.index")
ANSWER_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "answer_embeddings.pkl")
COURSE_SIMILARITY_PATH = os.path.join(chosen_model_dir, "course_similarity.pkl")
JOB_SIMILARITY_PATH = os.path.join(chosen_model_dir, "job_similarity.pkl")

# Global variables for precomputed data
tfidf_vectorizer = None
skill_tfidf = None
question_to_answer = None
faiss_index = None
answer_embeddings = None
course_similarity = None
job_similarity = None

# Improved dataset loading with fallback
def load_dataset(file_path, required_columns=None, additional_columns=None, fallback_data=None):
    required_columns = required_columns or ["Skill", "Question", "Answer"]
    additional_columns = additional_columns or ['popularity', 'completion_rate']
    try:
        df = pd.read_csv(file_path)
        missing_required = [col for col in required_columns if col not in df.columns]
        missing_additional = [col for col in additional_columns if col not in df.columns]
        
        if missing_required:
            logger.warning(f"Required columns {missing_required} missing in {file_path}. Adding empty values.")
            for col in missing_required:
                df[col] = ""
        
        if missing_additional:
            logger.warning(f"Additional columns {missing_additional} missing in {file_path}. Adding default values.")
            for col in missing_additional:
                df[col] = 0.8 if col == 'popularity' else 0.7 if col == 'completion_rate' else 0.0
        
        if 'level' not in df.columns:
            logger.warning(f"'level' column missing in {file_path}. Adding default 'Intermediate'.")
            df['level'] = 'Intermediate'
        else:
            df['level'] = df['level'].fillna('Intermediate')
        
        return df
    except Exception as e:
        logger.error(f"Error loading {file_path}: {e}. Using fallback data.")
        return pd.DataFrame(fallback_data) if fallback_data is not None else None

# Load datasets with fallbacks
questions_df = load_dataset("Generated_Skill-Based_Questions.csv", fallback_data={
    'Skill': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
    'Question': ['Advanced Linux question', 'Advanced Git question', 'Basic Node.js question', 
                 'Intermediate Python question', 'Basic Kubernetes question'],
    'Answer': ['Linux answer', 'Git answer', 'Node.js answer', 'Python answer', 'Kubernetes answer']
})

# Validate questions_df
if questions_df is None or questions_df.empty:
    logger.error("questions_df is empty or could not be loaded. Exiting.")
    exit(1)
if not all(col in questions_df.columns for col in ["Skill", "Question", "Answer"]):
    logger.error("questions_df is missing required columns. Exiting.")
    exit(1)
logger.info(f"questions_df loaded with {len(questions_df)} rows. Skills available: {list(questions_df['Skill'].unique())}")

# Load or Initialize Models with Fallback
def load_universal_model():
    default_model = "all-MiniLM-L6-v2"
    try:
        if os.path.exists(UNIVERSAL_MODEL_PATH):
            logger.info(f"Loading universal model from {UNIVERSAL_MODEL_PATH}")
            return SentenceTransformer(UNIVERSAL_MODEL_PATH)
        logger.info(f"Loading universal model: {default_model}")
        model = SentenceTransformer(default_model)
        model.save(UNIVERSAL_MODEL_PATH)
        return model
    except Exception as e:
        logger.error(f"Failed to load universal model {default_model}: {e}. Exiting.")
        exit(1)

universal_model = load_universal_model()

try:
    detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH if os.path.exists(DETECTOR_MODEL_PATH) else "roberta-base-openai-detector")
    detector_model = AutoModelForSequenceClassification.from_pretrained(DETECTOR_MODEL_PATH if os.path.exists(DETECTOR_MODEL_PATH) else "roberta-base-openai-detector")
except Exception as e:
    logger.error(f"Failed to load detector model: {e}. Exiting.")
    exit(1)

# Load Precomputed Resources
def load_precomputed_resources():
    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
    paths = [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH, ANSWER_EMBEDDINGS_PATH, COURSE_SIMILARITY_PATH, JOB_SIMILARITY_PATH]
    if all(os.path.exists(p) for p in paths):
        try:
            with open(TFIDF_PATH, 'rb') as f: tfidf_vectorizer = pickle.load(f)
            with open(SKILL_TFIDF_PATH, 'rb') as f: skill_tfidf = pickle.load(f)
            with open(QUESTION_ANSWER_PATH, 'rb') as f: question_to_answer = pickle.load(f)
            faiss_index = faiss.read_index(FAISS_INDEX_PATH)
            with open(ANSWER_EMBEDDINGS_PATH, 'rb') as f: answer_embeddings = pickle.load(f)
            with open(COURSE_SIMILARITY_PATH, 'rb') as f: course_similarity = pickle.load(f)
            with open(JOB_SIMILARITY_PATH, 'rb') as f: job_similarity = pickle.load(f)
            logger.info("Loaded precomputed resources successfully")
        except Exception as e:
            logger.error(f"Error loading precomputed resources: {e}")
            precompute_resources()
    else:
        precompute_resources()

# Precompute Resources Offline
def precompute_resources():
    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
    logger.info("Precomputing resources offline")
    try:
        tfidf_vectorizer = TfidfVectorizer(stop_words='english')
        all_texts = questions_df['Answer'].tolist() + questions_df['Question'].tolist()
        tfidf_vectorizer.fit(all_texts)
        
        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill]).toarray()[0] for skill in questions_df['Skill'].unique()}
        question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
        answer_embeddings = universal_model.encode(questions_df['Answer'].tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
        
        faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
        faiss_index.add(answer_embeddings)
        
        # Initialize course_similarity and job_similarity as empty dicts if not available
        course_similarity = course_similarity or {}
        job_similarity = job_similarity or {}
        
        with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
        with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
        with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
        faiss.write_index(faiss_index, FAISS_INDEX_PATH)
        with open(ANSWER_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(answer_embeddings, f)
        with open(COURSE_SIMILARITY_PATH, 'wb') as f: pickle.dump(course_similarity, f)
        with open(JOB_SIMILARITY_PATH, 'wb') as f: pickle.dump(job_similarity, f)
        universal_model.save(UNIVERSAL_MODEL_PATH)
        logger.info(f"Precomputed resources saved to {chosen_model_dir}")
    except Exception as e:
        logger.error(f"Error during precomputation: {e}")
        raise

# Evaluation with precomputed data
def evaluate_response(args):
    try:
        skill, user_answer, question_idx = args
        if not user_answer:
            return skill, 0.0, False

        inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512, padding=True)
        with torch.no_grad():
            logits = detector_model(**inputs).logits
        probs = scipy.special.softmax(logits, axis=1).tolist()[0]
        is_ai = probs[1] > 0.5

        user_embedding = universal_model.encode([user_answer], batch_size=1, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")[0]
        expected_embedding = torch.tensor(answer_embeddings[question_idx])
        score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100

        user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
        skill_vec = skill_tfidf.get(skill.lower(), np.zeros_like(user_tfidf))
        relevance = np.dot(user_tfidf, skill_vec) / (np.linalg.norm(user_tfidf) * np.linalg.norm(skill_vec) + 1e-10)
        score *= max(0.5, min(1.0, relevance))

        return skill, round(max(0, score), 2), is_ai
    except Exception as e:
        logger.error(f"Evaluation error for {skill}: {e}")
        return skill, 0.0, False

# Fetch questions for given skills
def get_questions_for_skills(skills):
    user_questions = []
    for skill in skills:
        skill = skill.strip().capitalize()  # Standardize skill format
        skill_questions = questions_df[questions_df['Skill'].str.capitalize() == skill]
        if not skill_questions.empty:
            user_questions.append(skill_questions.sample(1).iloc[0].to_dict())
        else:
            user_questions.append({
                'Skill': skill,
                'Question': f"What are the best practices for using {skill} in a production environment?",
                'Answer': f"Best practices for {skill} include proper documentation, monitoring, and security measures."
            })
    return user_questions

# Recommend courses from MongoDB
def recommend_courses_from_mongo(skills_to_improve, user_level, upgrade=False):
    try:
        if not skills_to_improve:
            return []

        target_level = 'Advanced' if upgrade else user_level
        query = {
            "skills": {"$in": skills_to_improve},
            "category": {"$regex": target_level, "$options": "i"}
        }
        courses = list(courses_collection.find(query).limit(3))
        return [{"title": course["title"], "provider": course.get("provider", "Unknown")} for course in courses]
    except Exception as e:
        logger.error(f"Course recommendation error: {e}")
        return []

# Recommend jobs from MongoDB
def recommend_jobs_from_mongo(user_skills, user_level):
    try:
        if not user_skills:
            return []

        query = {
            "skills": {"$in": user_skills},
            "status": "active"
        }
        jobs = list(jobs_collection.find(query).limit(5))
        return [{"jobTitle": job["jobTitle"], "companyName": job["companyName"], "location": job.get("location", "Remote")} for job in jobs]
    except Exception as e:
        logger.error(f"Job recommendation error: {e}")
        return []

# Flask application setup
app = Flask(__name__)

@app.route('/')
def health_check():
    return jsonify({"status": "active", "model_dir": chosen_model_dir})

@app.route('/get_questions', methods=['POST'])
def get_questions():
    try:
        data = request.get_json()
        if not data or 'skills' not in data:
            return jsonify({"error": "Missing skills field"}), 400

        user_skills = [s.strip() for s in data['skills'] if isinstance(s, str)]
        if not user_skills:
            return jsonify({"error": "No valid skills provided"}), 400

        load_precomputed_resources()
        questions = get_questions_for_skills(user_skills)
        return jsonify({"questions": questions})
    except Exception as e:
        logger.error(f"Get questions error: {e}")
        return jsonify({"error": "Internal server error"}), 500

@app.route('/assess', methods=['POST'])
def assess_skills():
    try:
        data = request.get_json()
        if not data or 'skills' not in data or 'answers' not in data or 'userId' not in data:
            return jsonify({"error": "Missing required fields"}), 400

        user_id = data['userId']
        user_skills = [s.strip() for s in data['skills'] if isinstance(s, str)]
        answers = [a.strip() for a in data['answers'] if isinstance(a, str)]
        user_level = data.get('user_level', 'Intermediate').strip()

        if not user_skills or len(answers) != len(user_skills):
            return jsonify({"error": "Answers count must match skills count"}), 400

        load_precomputed_resources()
        user_questions = get_questions_for_skills(user_skills)
        user_questions_df = pd.DataFrame(user_questions).reset_index(drop=True)

        user_responses = []
        for idx, row in user_questions_df.iterrows():
            answer = answers[idx]
            if not answer or answer.lower() == 'skip':
                user_responses.append((row['Skill'], None, None))
            else:
                question_idx = questions_df.index[questions_df['Question'] == row['Question']].tolist()
                if not question_idx:
                    logger.warning(f"Question not found in dataset: {row['Question']}")
                    user_responses.append((row['Skill'], None, None))
                    continue
                user_responses.append((row['Skill'], answer, question_idx[0]))

        results = [evaluate_response(response) for response in user_responses]

        user_scores = {}
        ai_flags = {}
        scores_list = []
        skipped_questions = [f"{skill} ({question})" for skill, user_code, _ in user_responses if not user_code]
        for skill, score, is_ai in results:
            if skill in user_scores:
                user_scores[skill] = max(user_scores[skill], score)
                ai_flags[skill] = ai_flags[skill] or is_ai
            else:
                user_scores[skill] = score
                ai_flags[skill] = is_ai
            if score > 0:
                scores_list.append(score)

        # Update user profile with scores
        skill_scores = [{"skill": skill, "score": score} for skill, score, _ in results if score > 0]
        users_collection.update_one(
            {"_id": user_id},
            {"$set": {"skillScores": skill_scores}},
            upsert=True
        )

        mean_score = np.mean(scores_list) if scores_list else 50
        dynamic_threshold = max(40, mean_score)
        weak_skills = [skill for skill, score in user_scores.items() if score > 0 and score < dynamic_threshold]

        courses = recommend_courses_from_mongo(weak_skills or user_skills, user_level, upgrade=not weak_skills)
        jobs = recommend_jobs_from_mongo(user_skills, user_level)

        return jsonify({
            "assessment_results": {
                "skills": [
                    {
                        "skill": skill,
                        "progress": f"{'■' * int(score//10)}{'-' * (10 - int(score//10))}",
                        "score": f"{score:.2f}%",
                        "origin": "AI-Generated" if is_ai else "Human-Written"
                    } for skill, score, is_ai in results
                ],
                "mean_score": round(mean_score, 2),
                "dynamic_threshold": round(dynamic_threshold, 2),
                "weak_skills": weak_skills,
                "skipped_questions": skipped_questions
            },
            "recommended_courses": courses,
            "recommended_jobs": jobs
        })
    except Exception as e:
        logger.error(f"Assessment error: {e}")
        return jsonify({"error": "Internal server error"}), 500

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=7860)