app.py

import streamlit as st
import os
import requests
from bs4 import BeautifulSoup
from sentence_transformers import SentenceTransformer, util
from transformers import pipeline

class URLValidator:
    """
    A production-ready URL validation class that evaluates the credibility of a webpage
    using multiple factors: domain trust, content relevance, fact-checking, bias detection, citations, and security.
    """

    def __init__(self):
        # API Keys
        self.serpapi_key = os.getenv('SERPAPI_KEY')
        self.google_safe_browsing_key = os.getenv('GOOGLE_SAFE_KEY')

        # Load models once to avoid redundant API calls
        self.similarity_model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
        self.fake_news_classifier = pipeline("text-classification", model="mrm8488/bert-tiny-finetuned-fake-news-detection")
        self.sentiment_analyzer = pipeline("text-classification", model="cardiffnlp/twitter-roberta-base-sentiment")

    def fetch_page_content(self, url: str) -> str:
        """ Fetches and extracts text content from the given URL. """
        try:
            response = requests.get(url, timeout=10)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, "html.parser")
            return " ".join([p.text for p in soup.find_all("p")])  # Extract paragraph text
        except requests.RequestException:
            return ""  # Fail gracefully by returning an empty string

    def get_domain_trust(self, url: str, content: str) -> int:
            """ Computes the domain trust score based on available data sources. """
            trust_scores = []

            # Hugging Face Fake News Detector
            if content:
                try:
                    trust_scores.append(self.get_domain_trust_huggingface(content))
                except:
                    pass

            # Compute final score (average of available scores)
            return int(sum(trust_scores) / len(trust_scores)) if trust_scores else 50

    def get_domain_trust_huggingface(self, content: str) -> int:
        """ Uses a Hugging Face fake news detection model to assess credibility. """
        if not content:
            return 50  # Default score if no content available
        result = self.fake_news_classifier(content[:512])[0]  # Process only first 512 characters
        return 100 if result["label"] == "REAL" else 30 if result["label"] == "FAKE" else 50

    def compute_similarity_score(self, user_query: str, content: str) -> int:
        """ Computes semantic similarity between user query and page content. """
        if not content:
            return 0
        return int(util.pytorch_cos_sim(self.similarity_model.encode(user_query), self.similarity_model.encode(content)).item() * 100)

    def check_facts(self, content: str) -> int:
        """ Cross-checks extracted content with Google Fact Check API. """
        if not content:
            return 50
        api_url = f"https://toolbox.google.com/factcheck/api/v1/claimsearch?query={content[:200]}"
        try:
            response = requests.get(api_url)
            data = response.json()
            return 80 if "claims" in data and data["claims"] else 40
        except:
            return 50  # Default uncertainty score

    def check_google_scholar(self, url: str) -> int:
        """ Checks Google Scholar citations using SerpAPI. """
        serpapi_key = self.serpapi_key
        params = {"q": url, "engine": "google_scholar", "api_key": serpapi_key}
        try:
            response = requests.get("https://serpapi.com/search", params=params)
            data = response.json()
            return min(len(data.get("organic_results", [])) * 10, 100)  # Normalize
        except:
            return 0  # Default to no citations

    def detect_bias(self, content: str) -> int:
        """ Uses NLP sentiment analysis to detect potential bias in content. """
        if not content:
            return 50
        sentiment_result = self.sentiment_analyzer(content[:512])[0]
        return 100 if sentiment_result["label"] == "POSITIVE" else 50 if sentiment_result["label"] == "NEUTRAL" else 30

    def get_star_rating(self, score: float) -> tuple:
        """ Converts a score (0-100) into a 1-5 star rating. """
        stars = max(1, min(5, round(score / 20)))  # Normalize 100-scale to 5-star scale
        return stars, "⭐" * stars

    def generate_explanation(self, domain_trust, similarity_score, fact_check_score, bias_score, citation_score, safe_browsing_score, final_score) -> str:
        """ Generates a human-readable explanation for the score. """
        reasons = []
        if domain_trust < 50:
            reasons.append("The source has low domain authority.")
        else:
            reasons.append("The source has high domain authority.")
        if similarity_score < 50:
            reasons.append("The content is not highly relevant to your query.")
        else:
            reasons.append("The content is highly relevant to your query.")
        if fact_check_score < 50:
            reasons.append("Limited fact-checking verification found.")
        else:
            reasons.append("High level of fact-checking verification found.")
        if bias_score < 50:
            reasons.append("Potential bias detected in the content.")
        else:
            reasons.append("No bias detected in the content.")
        if citation_score < 30:
            reasons.append("Few citations found for this content.")
        else:
            reasons.append("High level of citations found for this content.")
        if safe_browsing_score < 50:
            reasons.append("No malicious content detected.")
        else:
            reasons.append("Possible malicious content detected.")

        return " ".join(reasons) if reasons else "This source is highly credible and relevant."



    def check_google_safe_browsing(self, url: str) -> int:
        """
        Uses Google Safe Browsing API to check if a URL is malicious.
        Returns:
            - 100 if safe
            - 30 if flagged as potentially harmful
            - 10 if confirmed malicious
        """
        api_url = f"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={self.google_safe_browsing_key}"
        payload = {
            "client": {
                "clientId": "your-app",
                "clientVersion": "1.0"
            },
            "threatInfo": {
                "threatTypes": ["MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION"],
                "platformTypes": ["ANY_PLATFORM"],
                "threatEntryTypes": ["URL"],
                "threatEntries": [{"url": url}]
            }
        }

        try:
            response = requests.post(api_url, json=payload)
            data = response.json()
            if "matches" in data:
                return 10  # Malicious URL detected
            return 100  # Safe URL
        except:
            return 50  # Default score if API request fails

    def rate_url_validity(self, user_query: str, url: str) -> dict:
        """ Main function to evaluate the validity of a webpage. """
        content = self.fetch_page_content(url)

        domain_trust = self.get_domain_trust(url, content)
        similarity_score = self.compute_similarity_score(user_query, content)
        fact_check_score = self.check_facts(content)
        bias_score = self.detect_bias(content)
        citation_score = self.check_google_scholar(url)
        safe_browsing_score = self.check_google_safe_browsing(url)

        final_score = (
            (0.30 * domain_trust) +
            (0.30 * similarity_score) +
            (0.20 * fact_check_score) +
            (0.10 * bias_score) +
            (0.10 * citation_score) +
            (0.05 * safe_browsing_score)
        )

        stars, icon = self.get_star_rating(final_score)
        explanation = self.generate_explanation(domain_trust, similarity_score, fact_check_score, bias_score, citation_score, safe_browsing_score, final_score)

        return {
            "raw_score": {
                "Domain Trust": domain_trust,
                "Content Relevance": similarity_score,
                "Fact-Check Score": fact_check_score,
                "Bias Score": bias_score,
                "Citation Score": citation_score,
                "Safe Browsing Score": safe_browsing_score,
                "Final Validity Score": final_score
            },
            "stars": {
                "score": stars,
                "icon": icon
            },
            "explanation": explanation
        }


st.title("URL Validator")

# Input fields for user prompt and URL
user_prompt = st.text_input("Enter your query:", placeholder="e.g., What mods should I use on Volt Prime?")
url_to_check = st.text_input("Enter URL to validate:", placeholder="e.g., https://overframe.gg/items/arsenal/61/volt-prime/")

# Run validation on button click
if st.button("Validate URL"):
    if user_prompt and url_to_check:
        validator = URLValidator()
        result = validator.rate_url_validity(user_prompt, url_to_check)

        st.subheader("Validation Results")
        st.json(result)  # Display JSON object
    else:
        st.warning("Please enter both a query and a URL.")