Delete rlhf.py

rlhf.py

@@ -1,122 +0,0 @@
-import pandas as pd
-import numpy as np
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
-import warnings
-warnings.filterwarnings("ignore")
-
-# Load the human evaluation dataset
-df = pd.read_excel("final_comments_evaluations_latest.xlsx")
-
-# Initialize the Granite 3.2-2B-Instruct model and tokenizer (from your existing setup)
-model_name = "ibm-granite/granite-3.2-2b-instruct"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name)
-device = "cuda" if torch.cuda.is_available() else "cpu"
-model.to(device)
-
-# Define a simple reward model (mockup based on dataset metrics)
-# In practice, this would be the trained reward model from Stage 3
-def reward_model(paraphrase, original_scores):
-    # Mock reward calculation: adjust scores based on trends in the dataset
-    base_toxicity = original_scores["toxicity"]
-    base_empathy = original_scores["empathy"]
-    # Simulate improved paraphrasing: reduce toxicity, increase empathy
-    new_toxicity = max(0.1, base_toxicity - 0.2)  # Reduce toxicity
-    new_empathy = min(0.9, base_empathy + 0.1)   # Increase empathy
-    new_bias = original_scores["bias"]
-    new_hallucination = max(0.1, original_scores["hallucination"] - 0.1)
-    # Composite reward score (weights based on dataset analysis)
-    reward = 0.4 * new_empathy - 0.3 * new_toxicity - 0.2 * new_bias - 0.1 * new_hallucination
-    return reward, {"toxicity": new_toxicity, "empathy": new_empathy, "bias": new_bias, "hallucination": new_hallucination}
-
-# Function to generate a paraphrase using your existing paraphrasing logic
-def generate_paraphrase(comment, max_length=128):
-    prompt = (
-        "You are a content moderator tasked with rewriting toxic comments into neutral and constructive ones while maintaining the original meaning. "
-        "Follow these guidelines:\n"
-        "- Remove explicit hate speech, personal attacks, or offensive language.\n"
-        "- Keep the response neutral and professional.\n"
-        "- Ensure the rewritten comment retains the original intent but in a constructive tone.\n"
-        "- Match the length and brevity of the original toxic comment whenever possible. Keep the response short and to the point.\n\n"
-        "Examples:\n"
-        "Toxic: \"You're so dumb! You never understand anything!\"\n"
-        "Neutral: \"You might be misunderstanding this.\"\n"
-        "Toxic: \"This is the worst idea ever. Only an idiot would suggest this.\"\n"
-        "Neutral: \"I don’t think this idea works well.\"\n"
-        "Toxic: \"You’re useless.\"\n"
-        "Neutral: \"This isn’t helping much.\"\n"
-        "Toxic: \"Shut up.\"\n"
-        "Neutral: \"Let’s take a break from this.\"\n\n"
-        f"Now, rewrite this comment: \"{comment}\""
-    )
-    inputs = tokenizer(prompt, return_tensors="pt", max_length=max_length, truncation=True).to(device)
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=50,
-        num_beams=4,
-        early_stopping=True,
-        do_sample=False
-    )
-    paraphrase = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    # Clean up the output by removing the prompt part
-    paraphrase = paraphrase.replace(prompt, "").strip()
-    if paraphrase.startswith("Neutral: "):
-        paraphrase = paraphrase[len("Neutral: "):].strip()
-    return paraphrase
-
-# RLHF Loop
-max_iterations = 2
-reward_threshold = 0.2  # Target for acceptable paraphrases (based on dataset range -0.25 to 0.24)
-results = []
-
-for idx, row in df.iterrows():
-    original_comment = row["Comment"]
-    current_paraphrase = row["Paraphrase_Comment"]
-    current_reward = row["reward_score"]
-    current_scores = {
-        "toxicity": row["toxicity"],
-        "empathy": row["empathy"],
-        "bias": row["bias"],
-        "hallucination": row["hallucination"]
-    }
-    
-    best_paraphrase = current_paraphrase
-    best_reward = current_reward
-    best_scores = current_scores.copy()
-    
-    # Iteratively refine the paraphrase
-    for iteration in range(max_iterations):
-        # Generate a new paraphrase
-        new_paraphrase = generate_paraphrase(original_comment)
-        # Evaluate the new paraphrase with the reward model
-        new_reward, new_scores = reward_model(new_paraphrase, current_scores)
-        
-        # If the new reward is better, update the best paraphrase
-        if new_reward > best_reward:
-            best_paraphrase = new_paraphrase
-            best_reward = new_reward
-            best_scores = new_scores
-        
-        # Stop if the reward exceeds the threshold
-        if best_reward >= reward_threshold:
-            break
-    
-    # Store the result
-    results.append({
-        "Comment": original_comment,
-        "Original_Paraphrase": current_paraphrase,
-        "Refined_Paraphrase": best_paraphrase,
-        "Original_Reward_Score": current_reward,
-        "Refined_Reward_Score": best_reward,
-        "Refined_Empathy": best_scores["empathy"],
-        "Refined_Toxicity": best_scores["toxicity"],
-        "Refined_Bias": best_scores["bias"],
-        "Refined_Hallucination": best_scores["hallucination"],
-        "Human_Evaluation_Reasoning": row["Human_Evaluation_Reasoning"]
-    })
-
-# Save the results to a CSV file
-results_df = pd.DataFrame(results)
-results_df.to_csv("refined_paraphrases.csv", index=False)
-print("Refinement complete. Results saved to refined_paraphrases.csv")