# metrics.py import nltk from nltk.translate.bleu_score import sentence_bleu from rouge_score import rouge_scorer from model_loader import metrics_models # Download required NLTK data nltk.download('punkt') def compute_semantic_similarity(original, paraphrased): """ Compute semantic similarity between the original and paraphrased comment using Sentence-BERT. Returns a similarity score between 0 and 1. """ try: sentence_bert = metrics_models.load_sentence_bert() embeddings = sentence_bert.encode([original, paraphrased]) similarity = float(embeddings[0] @ embeddings[1].T) return round(similarity, 2) except Exception as e: print(f"Error computing semantic similarity: {str(e)}") return None def compute_emotion_shift(original, paraphrased): """ Compute the emotion shift between the original and paraphrased comment. Returns the original emotion, paraphrased emotion, and whether the shift is positive. """ try: emotion_classifier = metrics_models.load_emotion_classifier() original_emotions = emotion_classifier(original) paraphrased_emotions = emotion_classifier(paraphrased) # Get the top emotion for each original_emotion = max(original_emotions[0], key=lambda x: x['score'])['label'] paraphrased_emotion = max(paraphrased_emotions[0], key=lambda x: x['score'])['label'] # Define negative and positive emotions negative_emotions = ['anger', 'sadness', 'fear'] positive_emotions = ['joy', 'love', 'surprise'] # Determine if the shift is positive emotion_shift_positive = ( (original_emotion in negative_emotions and paraphrased_emotion in positive_emotions) or (original_emotion in negative_emotions and paraphrased_emotion not in negative_emotions) ) return original_emotion, paraphrased_emotion, emotion_shift_positive except Exception as e: print(f"Error computing emotion shift: {str(e)}") return None, None, None def compute_empathy_score(paraphrased): """ Compute an empathy score for the paraphrased comment (placeholder). Returns a score between 0 and 1. """ try: # Placeholder: Compute empathy based on word presence (e.g., "sorry", "understand") empathy_words = ["sorry", "understand", "care", "help", "support"] words = paraphrased.lower().split() empathy_count = sum(1 for word in words if word in empathy_words) score = empathy_count / len(words) if words else 0 return round(score, 2) except Exception as e: print(f"Error computing empathy score: {str(e)}") return None def compute_bleu_score(original, paraphrased): """ Compute the BLEU score between the original and paraphrased comment. Returns a score between 0 and 1. """ try: reference = [nltk.word_tokenize(original.lower())] candidate = nltk.word_tokenize(paraphrased.lower()) score = sentence_bleu(reference, candidate, weights=(0.25, 0.25, 0.25, 0.25)) return round(score, 2) except Exception as e: print(f"Error computing BLEU score: {str(e)}") return None def compute_rouge_score(original, paraphrased): """ Compute ROUGE scores (ROUGE-1, ROUGE-2, ROUGE-L) between the original and paraphrased comment. Returns a dictionary with ROUGE scores. """ try: scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True) scores = scorer.score(original, paraphrased) return { 'rouge1': round(scores['rouge1'].fmeasure, 2), 'rouge2': round(scores['rouge2'].fmeasure, 2), 'rougeL': round(scores['rougeL'].fmeasure, 2) } except Exception as e: print(f"Error computing ROUGE scores: {str(e)}") return None def compute_entailment_score(original, paraphrased): """ Compute the entailment score to check factual consistency using an NLI model. Returns a score between 0 and 1. """ try: nli_classifier = metrics_models.load_nli_classifier() result = nli_classifier( original, paraphrased, candidate_labels=["entailment", "contradiction", "neutral"] ) entailment_score = next(score for label, score in zip(result['labels'], result['scores']) if label == "entailment") return round(entailment_score, 2) except Exception as e: print(f"Error computing entailment score: {str(e)}") return None