1st version of demo

app.py

@@ -1,400 +1,22 @@
-import warnings
+#display a data
+import gradio as gr
 
-import torchvision.transforms as transforms
-from google_img_source_search import ReverseImageSearcher
+def data_display(replace_df):
+    return "aaaa"
 
-# from src.images.CNN_model_classifier import predict_cnn
-# from src.images.diffusion_model_classifier import (
-#     ImageClassifier,
-#     predict_single_image,
-# )
-
-warnings.simplefilter(
-    action="ignore",
-    category=FutureWarning,
-)  # disable FutureWarning
-
-import gradio as gr  # noqa: E402
-from transformers import (  # noqa: E402
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
-    pipeline,
-)
-
-from src.texts.MAGE.deployment import (  # noqa: E402
-    detect,
-    preprocess,
-)
-from src.texts.PASTED.pasted_lexicon import Detector  # noqa: E402
-from src.texts.Search_Text.search import (  # noqa: E402
-    get_important_sentences,
-    get_keywords,
-    is_human_written,
-)
-from src.images.Search_Image.search import (
-    compare_images,
-    get_image_from_path,
-    get_image_from_url,
-)
-
-
-def convert_score_range(score):
-    """
-    Converts a score from the range [0, 1] to [-1, 1].
-
-    Args:
-    score: The original score in the range [0, 1].
-
-    Returns:
-    The converted score in the range [-1, 1].
-    """
-
-    return 2 * score - 1
-
-
-def generate_highlighted_text(text_scores):
-    """
-    Generates a highlighted text string based on the given text and scores.
-
-    Args:
-        text_scores: A list of tuples, where each tuple contains a text
-            segment and its score.
-
-    Returns:
-        A string of HTML code with highlighted text.
-    """
-    highlighted_text = ""
-    for text, score in text_scores:
-        # Map score to a color using a gradient
-        color = f"rgba(255, 0, 0, {1 - score})"  # Red to green gradient
-        highlighted_text += (
-            f"<span style='background-color: {color}'>{text}</span>"  # noqa
-        )
-    return highlighted_text
-
-
-def separate_characters_with_mask(text, mask):
-    """Separates characters in a string and pairs them with a mask sign.
-
-    Args:
-        text: The input string.
-
-    Returns:
-        A list of tuples, where each tuple contains a character and a mask.
-    """
-
-    return [(char, mask) for char in text]
-
-
-def detect_ai_text(model_name, search_engine, text):
-    if search_engine is True:
-        keywords = get_keywords(text)
-        important_sentences = get_important_sentences(text, keywords)
-        predictions = is_human_written(important_sentences[0])
-        print("keywords: ", keywords)
-        print("important_sentences: ", important_sentences)
-        print("predictions: ", predictions)
-        if predictions == -1:
-            caption = "[Found exact match] "
-            text_scores = list(zip([caption, text], [0, predictions]))
-            print("text_scores: ", text_scores)
-            return text_scores
-
-    if model_name == "SimLLM":
-        tokenize_input = SimLLM_tokenizer(text, return_tensors="pt")
-        outputs = SimLLM_model(**tokenize_input)
-        predictions = outputs.logits.argmax(dim=-1).item()
-        if predictions == 0:
-            predictions = "human-written"
-        else:
-            predictions = "machine-generated"
-
-    elif model_name == "MAGE":
-        processed_text = preprocess(text)
-        predictions = detect(
-            processed_text,
-            MAGE_tokenizer,
-            MAGE_model,
-            device,
-        )
-
-    elif model_name == "chatgpt-detector-roberta":
-        predictions = roberta_pipeline_en(text)[0]["label"]
-        if predictions == "Human":
-            predictions = "human-written"
-        else:  # ChatGPT
-            predictions = "machine-generated"
-    elif model_name == "PASTED-Lexical":
-        predictions = detector(text)
-
-    if model_name != "PASTED-Lexical":
-        text_scores = list(zip([text], [predictions]))
-    else:
-        text_scores = []
-        for text, score in predictions:
-            new_score = convert_score_range(score)  # normalize score
-            text_scores.append((text, new_score))
-
-    return text_scores
-
-
-diffusion_model_path = (
-    "src/images/Diffusion/model_checkpoints/"
-    "image-classifier-step=7007-val_loss=0.09.ckpt"
-)
-cnn_model_path = "src/images/CNN/model_checkpoints/blur_jpg_prob0.5.pth"
-
-
-def detect_ai_image(input_image_path, search_engine):
-    # if search_engine is True:
-        # Search image
-        
-        rev_img_searcher = ReverseImageSearcher()
-        search_items = rev_img_searcher.search_by_file(input_image_path)
-        min_result_difference = 5000
-        result_image_url = ""
-        input_image = get_image_from_path(input_image_path)
-        
-        for search_item in search_items:
-            # print(f'Title: {search_item.page_title}')
-            # print(f'Site: {search_item.page_url}')
-            # print(f'Img: {search_item.image_url}\n')
-
-            # Compare each search result image with the input image
-            result_image = get_image_from_url(search_item.image_url)
-            # input_image = get_image_from_url(search_item.image_url)
-            result_difference = compare_images(result_image, input_image)
-            
-            print(f"Difference with search result: {result_difference}")
-            print(f"Result image url: {search_item.page_url}\n")
-
-            if min_result_difference > result_difference:
-                min_result_difference = result_difference
-                result_image_url = search_item.image_url
-                result_page_url = search_item.page_url
-                
-                
-            if result_difference == 0:
-                break
-
-
-        if min_result_difference == 0:
-            result = f"<h1>Input image is LIKELY SIMILAR to image from:</h1>"\
-                    f"<ul>"\
-                    f'<li>\nPage URL: <a href="url">{result_page_url}</a></li>'\
-                    f'<li>\nImage URL: <a href="url">{result_image_url}</a></li>'\
-                    f"<li>\nDifference score: {min_result_difference}</li>"\
-                    f"</ul>"
-        elif 10 > min_result_difference > 0:
-            result = f"<h1>Input image is potentially a VARIATRION from:</h1>"\
-                    f"<ul>"\
-                    f'<li>\nPage URL: <a href="url">{result_page_url}</a></li>'\
-                    f'<li>\nImage URL: <a href="url">{result_image_url}</a></li>'\
-                    f"<li>\nDifference score: {min_result_difference}</li>"\
-                    f"</ul>"
-        elif min_result_difference < 5000:
-            result = f"<h1>Input image is not similar to any search results.</h1>"\
-                    f"<ul>"\
-                    f'<li>\nPage URL: <a href="url">{result_page_url}</a></li>'\
-                    f'<li>\nImage URL: <a href="url">{result_image_url}</a></li>'\
-                    f"<li>\nDifference score: {min_result_difference}</li>"\
-                    f"</ul>"
-        else:
-            result = f"<h1>No search result found.</h1>"\
-        
-        return result
-
-    # def get_prediction_diffusion(image):
-    #     model = ImageClassifier.load_from_checkpoint(diffusion_model_path)
-
-    #     prediction = predict_single_image(image, model)
-    #     return (prediction >= 0.5, prediction)
-
-    # def get_prediction_cnn(image):
-    #     prediction = predict_cnn(image, cnn_model_path)
-    #     return (prediction >= 0.5, prediction)
-
-    # # Define the transformations for the image
-    # transform = transforms.Compose(
-    #     [
-    #         transforms.Resize((224, 224)),  # Image size expected by ResNet50
-    #         transforms.ToTensor(),
-    #         transforms.Normalize(
-    #             mean=[0.485, 0.456, 0.406],
-    #             std=[0.229, 0.224, 0.225],
-    #         ),
-    #     ],
-    # )
-    # image_tensor = transform(inp)
-    # pred_diff, prob_diff = get_prediction_diffusion(image_tensor)
-    # pred_cnn, prob_cnn = get_prediction_cnn(image_tensor)
-    # verdict = (
-    #     "AI Generated" if (pred_diff or pred_cnn) else "No GenAI detected"
-    # )
-    # return (
-    #     f"<h1>{verdict}</h1>"
-    #     f"<ul>"
-    #     f"<li>Diffusion detection score: {prob_diff:.1%} "
-    #     f"{'(MATCH)' if pred_diff else ''}</li>"
-    #     f"<li>CNN detection score: {prob_cnn:.1%} "
-    #     f"{'(MATCH)' if pred_cnn else ''}</li>"
-    #     f"</ul>"
-    # )
-
-
-# Define GPUs
-device = "cpu"  # use 'cuda:0' if GPU is available
-
-# init MAGE
-model_dir = "yaful/MAGE"  # model in huggingface
-MAGE_tokenizer = AutoTokenizer.from_pretrained(model_dir)
-MAGE_model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(
-    device,
-)
-
-# init chatgpt-detector-roberta
-model_dir = "Hello-SimpleAI/chatgpt-detector-roberta"  # model in huggingface
-roberta_pipeline_en = pipeline(task="text-classification", model=model_dir)
-
-# init PASTED
-model_dir = "linzw/PASTED-Lexical"
-detector = Detector(model_dir, device)
-
-# init SimLLM
-model_path = "./models/single_model_detector"
-SimLLM_tokenizer = AutoTokenizer.from_pretrained(model_path)
-SimLLM_model = AutoModelForSequenceClassification.from_pretrained(model_path)
-
-# Init variable for UI
-title = """
-<center>
-
-<h1> AI-generated content detection </h1>
-<b> Demo by NICT & Tokyo Techies <b>
-
-</center>
-"""
-
-examples = [
-    [
-        "SimLLM",
-        False,
-        """\
-The BBC's long-running consumer rights series Watchdog is to end as a \
-standalone programme, instead becoming part of The One Show. Watchdog \
-began in 1980 as a strand of Nationwide, but proved so popular it \
-became a separate programme in 1985. Co-host Steph McGovern has moved \
-to Channel 4, but Matt Allwright and Nikki Fox will stay to front the \
-new strand. The BBC said they would investigate viewer complaints all \
-year round rather than for two series a year.
-""",
-    ],
-    [
-        "chatgpt-detector-roberta",
-        False,
-        """\
-Artificial intelligence (AI) is the science of making machines \
-intelligent. It enables computers to learn from data, recognize \
-patterns, and make decisions. AI powers many technologies we use \
-daily, from voice assistants to self-driving cars. It's rapidly \
-evolving, promising to revolutionize various industries and reshape \
-the future.""",
-    ],
-]
-
-model_remark = """<left>
-Model sources:
-<a href="https://github.com/Tokyo-Techies/prj-nict-ai-content-detection">SimLLM</a>,
-<a href="https://github.com/yafuly/MAGE">MAGE</a>,
-<a href="https://huggingface.co/Hello-SimpleAI/chatgpt-detector-roberta">chatgpt-detector-roberta</a>,
-<a href="https://github.com/Linzwcs/PASTED">PASTED-Lexical</a>.
-</left>
-"""  # noqa: E501
-
-image_samples = [
-    ["src/images/samples/fake_dalle.jpg", "Generated (Dall-E)"],
-    ["src/images/samples/fake_midjourney.png", "Generated (MidJourney)"],
-    ["src/images/samples/fake_stable.jpg", "Generated (Stable Diffusion)"],
-    ["src/images/samples/fake_cnn.png", "Generated (GAN)"],
-    ["src/images/samples/real.png", "Organic"],
-    [
-        "https://p.potaufeu.asahi.com/1831-p/picture/27695628/89644a996fdd0cfc9e06398c64320fbe.jpg",  # noqa E501
-        "Internet GenAI",
-    ],
-]
-image_samples_path = [i[0] for i in image_samples]
-
-# UI
 with gr.Blocks() as demo:
-    with gr.Row():
-        gr.HTML(title)
-    with gr.Row():
-        with gr.Tab("Text"):
-            with gr.Row():
-                with gr.Column():
-                    model = gr.Dropdown(
-                        [
-                            "SimLLM",
-                            "MAGE",
-                            "chatgpt-detector-roberta",
-                            "PASTED-Lexical",
-                        ],
-                        label="Detection model",
-                    )
-                    search_engine = gr.Checkbox(label="Use search engine")
-                    gr.HTML(model_remark)
-                with gr.Column():
-                    text_input = gr.Textbox(
-                        label="Input text",
-                        placeholder="Enter text here...",
-                        lines=5,
-                    )
-
-            output = gr.HighlightedText(
-                label="Detection results",
-                combine_adjacent=True,
-                show_legend=True,
-                color_map={
-                    "human-written": "#7d58cf",
-                    "machine-generated": "#e34242",
-                },
-            )
-
-            gr.Examples(
-                examples=examples,
-                inputs=[model, search_engine, text_input],
-            )
-            model.change(
-                detect_ai_text,
-                inputs=[model, search_engine, text_input],
-                outputs=output,
-            )
-            search_engine.change(
-                detect_ai_text,
-                inputs=[model, search_engine, text_input],
-                outputs=output,
-            )
-            text_input.change(
-                detect_ai_text,
-                inputs=[model, search_engine, text_input],
-                outputs=output,
-            )
-        with gr.Tab("Images"):
-            with gr.Row():
-                input_image = gr.Image(type="filepath")
-                with gr.Column():
-                    output_image = gr.Markdown(height=400)
-            gr.Examples(
-                examples=image_samples,
-                inputs=input_image,
-            )
-
-            input_image.change(
-                detect_ai_image,
-                inputs=input_image,
-                outputs=output_image,
-            )
-
-
-# demo.launch(share=True)
-demo.launch(allowed_paths=image_samples_path, share=True)
+    replace_df = gr.Dataframe(
+        # headers=["Find what:", "Replace with:"],
+        # datatype=["str", "str"],
+        # row_count=(1, "dynamic"),
+        # col_count=(2, "fixed"),
+        # interactive=True
+    )
+    replace_button = gr.Button("Replace all")
+    news_content = gr.Textbox(label="Content", value="", lines=12)
+    
+    
+    replace_button.click(data_display, 
+                        inputs=[replace_df], 
+                        outputs=[news_content])
+demo.launch()

application.py

@@ -6,13 +6,28 @@ import requests
 from PIL import Image
 import re
 
+from src.application.content_detection import generate_analysis_report
 from src.application.url_reader import URLReader
+from src.application.content_generation import generate_content, replace_text
+
+# from dotenv import load_dotenv
+
+# load_dotenv()
+# AZURE_OPENAI_API_KEY = os.getenv('AZURE_OPENAI_API_KEY')
+# AZURE_OPENAI_ENDPOINT = os.getenv('AZURE_OPENAI_ENDPOINT')
+# AZURE_OPENAI_API_VERSION = os.getenv('AZURE_OPENAI_API_VERSION')
+
+# client = openai.AzureOpenAI(
+#     api_version = AZURE_OPENAI_API_VERSION,
+#     api_key = AZURE_OPENAI_API_KEY,  
+#     azure_endpoint = AZURE_OPENAI_ENDPOINT,
+#     )
 
-OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
-openai.api_key = os.getenv('OPENAI_API_KEY')
 GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
 SEARCH_ENGINE_ID = os.getenv('SEARCH_ENGINE_ID')
 
+AZURE_OPENAI_MODEL = ["gpt-4o-mini", "gpt-4o"]
+
 def load_url(url):
     """
     Load content from the given URL.
@@ -39,99 +54,75 @@ def load_url(url):
 
     return content.title, content.text, image
 
-
-def replace_terms(text, input_term, destination_term):
-    # Replace input_term with destination_term in the text
-    modified_text = re.sub(input_term, destination_term, text) 
-    return modified_text
-
-def generate_content(model1, model2, title, content):
-    # Generate text using the selected models
-    full_content = ""
-    input_type = ""
-    if title and content:
-        full_content = title + "\n" + content
-        input_type = "title and content"
-    elif title:
-        full_content = title
-        input_type = "title"
-    elif content:
-        full_content = title
-        input_type = "content"
-        
-def generate_text(model, full_context, input_type):
-    # Generate text using the selected model
-    if input_type == "":
-        prompt = "Generate a random fake news article"
-    else:
-        prompt = f"Generate a fake news article (title and content) based on the following {input_type}: {full_context}"
-        
-    try:
-        response = openai.ChatCompletion.create(
-        model=model, 
-        messages=[
-            {"role": "user", "content": prompt}
-        ]
-        )
-        return response.choices[0].message.content
-
-    except openai.error.OpenAIError as e:
-        print(f"Error interacting with OpenAI API: {e}")
-        return "An error occurred while processing your request."
+def show_detailed_analysis(title):
+    return f"More details of {title} will be shown here."
 
 # Define the GUI
 with gr.Blocks() as demo:
-    gr.Markdown("# Fake News Detection")
+    gr.Markdown("# FAKE NEWS DETECTION")
 
     with gr.Row():
+        # SETTINGS 
         with gr.Column(scale=1):
-            gr.Markdown("## Settings")
-            gr.Markdown("This tool generates fake news by modifying the content of a given URL.")
-
-            with gr.Accordion("1. Enter a URL"):
-                #gr.Markdown("  1. Enter a URL.")
-                url_input = gr.Textbox(
-                    label="URL",
-                    value="https://bbc.com/future/article/20250110-how-often-you-should-wash-your-towels-according-to-science",
+            with gr.Accordion("Settings"):
+                gr.Markdown("This tool generates fake news by modifying the content of a given URL.")
+
+                with gr.Accordion("1. Enter a URL"):
+                    url_input = gr.Textbox(
+                        label="URL",
+                        value="https://bbc.com/future/article/20250110-how-often-you-should-wash-your-towels-according-to-science",
+                        )
+                    load_button = gr.Button("Load URL")
+                    
+                with gr.Accordion("2. Select a content-generation model", open=True):
+                    with gr.Row():
+                        text_generation_model = gr.Dropdown(choices=AZURE_OPENAI_MODEL, label="Text-generation model")
+                        image_generation_model = gr.Dropdown(choices=["Dall-e", "Stable Diffusion"], label="Image-generation model")
+                    generate_button = gr.Button("Random generation")
+
+                with gr.Accordion("3. Replace any terms", open=True):
+                    replace_df = gr.Dataframe(
+                        headers=["Find what:", "Replace with:"],
+                        datatype=["str", "str"],
+                        row_count=(1, "dynamic"),
+                        col_count=(2, "fixed"),
+                        interactive=True
                     )
-                load_button = gr.Button("Load an URL...")
-                
-            with gr.Accordion("2. Select a content-generation model", open=True):
-                with gr.Row():
-                    model1_dropdown = gr.Dropdown(choices=["GPT 4o", "GPT 4o-mini"], label="Text-generation model")
-                    model2_dropdown = gr.Dropdown(choices=["Dall-e", "Stable Diffusion"], label="Image-generation model")
-                generate_button = gr.Button("Random generation...")
-
-            with gr.Accordion("3. Replace any terms", open=True):
-                with gr.Row():
-                    input_term_box = gr.Textbox(label="Input Term")
-                    destination_term_box = gr.Textbox(label="Destination Term")
-                replace_button = gr.Button("Replace term...")
-            
-            process_button = gr.Button("Process")
+                    replace_button = gr.Button("Replace all")
 
-        with gr.Column(scale=2):
-            gr.Markdown("## News contents")
-            title_input = gr.Textbox(label="Title", value="")
-            with gr.Row():
-                image_view = gr.Image(label="Image") 
-                content_input = gr.Textbox(label="Content", value="", lines=15)
+        # GENERATED CONTENT
+        with gr.Column(scale=1):
+            with gr.Accordion("Generated News Contents"):
+                detection_button = gr.Button("Check for fake news")
+                news_title = gr.Textbox(label="Title", value="")
+                news_image = gr.Image(label="Image") 
+                news_content = gr.Textbox(label="Content", value="", lines=12)
 
-    
+        # FAKE NEWS ANALYSIS REPORT
+        with gr.Column(scale=1):
+            with gr.Accordion("Fake News Analysis"):
+                html_out = gr.HTML() 
+                detailed_analysis_button = gr.Button("Show detailed analysis...")
 
     # Connect events
     load_button.click(
         load_url, 
         inputs=url_input, 
-        outputs=[title_input, content_input, image_view]
+        outputs=[news_title, news_content, news_image]
         )
-    replace_button.click(replace_terms, 
-                        inputs=[content_input, input_term_box, destination_term_box], 
-                        outputs=content_input)
-    process_button.click(generate_text, 
-                        inputs=[url_input, model1_dropdown, model2_dropdown, input_term_box, destination_term_box, title_input, content_input], 
-                        outputs=[title_input, content_input])
-
+    replace_button.click(replace_text, 
+                        inputs=[news_title, news_content, replace_df], 
+                        outputs=[news_title, news_content])
+    generate_button.click(generate_content, 
+                        inputs=[text_generation_model, image_generation_model, news_title, news_content], 
+                        outputs=[news_title, news_content])
+    detection_button.click(generate_analysis_report,
+                           inputs=[news_title, news_content, news_image],
+                           outputs=html_out) 
+    detailed_analysis_button.click(show_detailed_analysis,
+                                   inputs=[news_title],
+                                   outputs=[html_out])
+    # change Image
     #url_input.change(load_image, inputs=url_input, outputs=image_view) 
 
 demo.launch()

requirements.txt

@@ -1,7 +1,10 @@
 # UI
 gradio==5.13.1
 
-# network
+# Read environment variables
+python-dotenv
+
+# URL processing
 requests==2.31.0
 beautifulsoup4==4.12.3
 lxml-html-clean==0.4.1

src/application/content_detection.py

@@ -0,0 +1,112 @@
+
+
+from src.application.text.model_detection import detect_by_ai_model
+from src.application.text.search_detection import check_human, detect_by_relative_search
+
+
+def determine_text_origin(title, content):
+    """
+    Determines the origin of the given text based on paraphrasing detection and human authorship analysis.
+
+    Args:
+        text: The input text to be analyzed.
+
+    Returns:
+        str: The predicted origin of the text: 
+             - "HUMAN": If the text is likely written by a human.
+             - "MACHINE": If the text is likely generated by a machine.
+    """
+    # Classify by search engine
+    text = title + "\n\n" + content
+    is_paraphrased, referent_url, aligned_sentences = detect_by_relative_search(text)
+    prediction_score = 0.0
+    if not is_paraphrased:
+        prediction_label = "UNKNOWN"
+    else:
+        prediction_score = 100.0
+        if check_human(aligned_sentences):
+            prediction_label = "HUMAN"
+        else:
+            prediction_label = "MACHINE"
+    
+    if prediction_label == "UNKNOWN":
+        # Classify by SOTA model
+        prediction_label, prediction_score = detect_by_ai_model(text)
+    
+    return prediction_label, prediction_score, referent_url
+
+
+def generate_analysis_report(news_title, news_content, news_image):
+    
+    text_prediction_label, text_confidence_score, text_referent_url = determine_text_origin(news_title, news_content)
+    
+    # Analyze text content
+    url1 = text_referent_url
+    #url2 = "https://example.com/article2"
+    
+    # Forensic analysis
+    if text_prediction_label == "MACHINE":
+        text_prediction_label = "The text is modified by GPT-4o (AI)"
+    else:
+        text_prediction_label = "The text is written by HUMAN"
+    
+    image_detection_results = "MACHINE"
+    if image_detection_results == "MACHINE":
+        image_detection_results = "The image is generated by Dall-e (AI)"
+    else:
+        image_detection_results = "The image is generated by HUMAN"
+    image_confidence_score = 90.5
+    
+    news_detection_results = "MACHINE"
+    if news_detection_results == "MACHINE":
+        news_detection_results = "The whole news generated by AI"
+    else:
+        news_detection_results = "The whole news written by HUMAN"
+    news_confidence_score = 97.4
+    
+    # Misinformation analysis
+    out_of_context_results = "cohesive"
+    if out_of_context_results == "cohesive":
+        out_of_context_results = "The input news is cohesive (non-out-of-context)"
+    else:
+        out_of_context_results = "The input news is out-of-context"
+    out_of_context_confidence_score = 96.7
+    
+    # Description
+    description = "The description should be concise, clear, and aimed at helping general readers understand the case." 
+    
+    html_template = f"""
+    <h2>Placeholder for results</h2>
+    
+    <div>
+        <h3>Originality:</h3>
+        <ul>
+            <li><a href="{url1}" target="_blank">{url1[:40] + "..."}</a></li>
+        </ul>
+    </div>
+
+    <div>
+        <h3>Forensic:</h3>
+        <b>{news_detection_results} (confidence = {news_confidence_score}%)</b>
+        <ul>
+            <li>{text_prediction_label} (confidence = {text_confidence_score}%)</li>
+            <li>{image_detection_results} (confidence = {image_confidence_score}%)</li>
+        </ul>
+    </div>
+
+    <div>
+        <h3>Misinformation:</h3>
+        <ul>
+            <li>The input news is {out_of_context_results} (confidence = {out_of_context_confidence_score}%)</li>
+        </ul>
+    </div>
+
+    <div>
+        <h3>Description (optional):</h3>
+        <ul>
+            <li>{description}</li>
+        </ul>
+    </div>
+    """
+
+    return html_template

src/application/content_generation.py

@@ -0,0 +1,71 @@
+import openai
+from dotenv import load_dotenv
+import os
+import re
+
+load_dotenv()
+AZURE_OPENAI_API_KEY = os.getenv('AZURE_OPENAI_API_KEY')
+AZURE_OPENAI_ENDPOINT = os.getenv('AZURE_OPENAI_ENDPOINT')
+AZURE_OPENAI_API_VERSION = os.getenv('AZURE_OPENAI_API_VERSION')
+
+client = openai.AzureOpenAI(
+    api_version = AZURE_OPENAI_API_VERSION,
+    api_key = AZURE_OPENAI_API_KEY,  
+    azure_endpoint = AZURE_OPENAI_ENDPOINT,
+    )
+
+def generate_content(text_generation_model, image_generation_model, title, content):
+    # Generate text using the selected models
+    full_content = ""
+    input_type = ""
+    if title and content:
+        full_content = title + "\n" + content
+        input_type = "title and content"
+    elif title:
+        full_content = title
+        input_type = "title"
+    elif content:
+        full_content = title
+        input_type = "content"
+        
+    # Generate text using the text generation model
+    generated_text = generate_text(text_generation_model, full_content, input_type)
+    return title, generated_text
+        
+def generate_text(model, full_context, input_type):
+    # Generate text using the selected model
+    if input_type == "":
+        prompt = "Generate a random fake news article"
+    else:
+        prompt = f"Generate a fake news article (title and content) based on the following: # Title: {input_type}:\n\n# Content: {full_context}"
+        
+    try:
+        response = client.chat.completions.create(
+            model=model, 
+            messages = [{"role": "system", "content": prompt}],
+        )
+        
+        print("Response from OpenAI API: ", response.choices[0].message.content)
+        return response.choices[0].message.content
+
+    except openai.OpenAIError as e:
+        print(f"Error interacting with OpenAI API: {e}")
+        return "An error occurred while processing your request."
+    
+def replace_text(news_title, news_content, replace_df):
+    """
+    Replaces occurrences in the input text based on the provided DataFrame.
+
+    Args:
+        text: The input text.
+        replace_df: A pandas DataFrame with two columns: "find_what" and "replace_with".
+
+    Returns:
+        The text after all replacements have been made.
+    """
+    for _, row in replace_df.iterrows():
+        find_what = row["Find what:"]
+        replace_with = row["Replace with:"]
+        news_content = news_content.replace(find_what, replace_with)
+        news_title = news_title.replace(find_what, replace_with)
+    return news_title, news_content

src/application/text/helper.py

@@ -0,0 +1,153 @@
+from collections import Counter
+import re
+import string
+from sklearn.feature_extraction.text import TfidfVectorizer
+from nltk.tokenize import word_tokenize
+from nltk.util import ngrams
+
+
+def clean_text(text):
+    """Doc cleaning"""
+    punctuations = r"""!"#$%&'()*+-/:;<=>?@[\]^_`{|}~"""  # not include , and . due to number
+    # Lowering text
+    text = text.lower()
+    
+    # Removing punctuation
+    text = "".join([c for c in text if c not in punctuations])
+    
+    # Removing whitespace and newlines
+    text = re.sub(r'\s+',' ',text)
+    
+    text.replace("£", " * ")
+    
+    words = text.split()
+    text = ' '.join(words[:18])  # Join the first 18 words back into a string
+    
+    return text
+
+def remove_punctuation(text):
+    """Remove punctuation from a given text."""
+    punctuation_without_dot = string.punctuation.replace(".", "")
+    translator = str.maketrans('', '', punctuation_without_dot)
+    return text.translate(translator)
+
+def get_keywords(text, num_keywords=5):
+    """Return top k keywords from a doc using TF-IDF method"""
+    
+    # Create a TF-IDF Vectorizer
+    vectorizer = TfidfVectorizer(stop_words='english')
+    
+    # Fit and transform the text
+    tfidf_matrix = vectorizer.fit_transform([text])
+    
+    # Get feature names (words)
+    feature_names = vectorizer.get_feature_names_out()
+    
+    # Get TF-IDF scores
+    tfidf_scores = tfidf_matrix.toarray()[0]
+    
+    # Sort words by TF-IDF score
+    word_scores = list(zip(feature_names, tfidf_scores))
+    word_scores.sort(key=lambda x: x[1], reverse=True)
+    
+    # Return top keywords
+    return [word for word, score in word_scores[:num_keywords]]
+
+"""
+# Example usage
+text = "Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to natural intelligence displayed by animals including humans. Leading AI textbooks define the field as the study of "intelligent agents": any system that perceives its environment and takes actions that maximize its chance of achieving its goals. Some popular accounts use the term "artificial intelligence" to describe machines that mimic "cognitive" functions that humans associate with the human mind, such as "learning" and "problem solving", however this definition is rejected by major AI researchers."
+print(f"\n# Input text:\n'{text}'")
+print("\n----------------------\n") 
+
+keywords = get_keywords(text)
+print("# Top keywords:", keywords)
+print("\n----------------------\n")
+"""
+
+def get_important_sentences(paragraph: str, keywords: list[str], num_sentences: int = 3) -> list[str]:
+    """
+    Selects important sentences from a given paragraph based on a list of keywords.
+
+    Args:
+        paragraph (str): The input paragraph.
+        keywords (list[str]): List of important keywords.
+        num_sentences (int): Number of sentences to return (default is 3).
+
+    Returns:
+        list: A list of important sentences.
+    """
+    # Clean and split the paragraph into sentences
+    sentences = [s.strip() for s in re.split(r'(?<=[.!?])\s+', paragraph) if s.strip()]
+    
+    # Calculate the importance score for each sentence
+    sentence_scores = []
+    for sentence in sentences:
+        processed_sentence = clean_text(sentence)
+        score = 0
+        words = processed_sentence.lower().split()
+        word_count = Counter(words)
+        
+        for keyword in keywords:
+            if keyword.lower() in word_count:
+                score += word_count[keyword.lower()]
+        
+        sentence_scores.append((sentence, score))
+    
+    # Sort sentences by their scores in descending order
+    sentence_scores.sort(key=lambda x: x[1], reverse=True)
+    
+    # Return the top N sentences
+    return [sentence for sentence, score in sentence_scores[:num_sentences]]
+
+"""# Example usage
+keywords = get_keywords(paragraph)
+important_sentences = get_important_sentences(paragraph, keywords)
+
+print("# Important sentences:")
+for i, sentence in enumerate(important_sentences, 1):
+    print(f"{i}. {sentence}")
+print("\n----------------------\n") 
+"""
+
+def extract_important_phrases(paragraph: str, keywords: list[str], phrase_length: int = 5) -> list[str]:
+    """
+    Extracts important phrases from a given paragraph based on a list of keywords.
+    Phrase length is auto-determined, and overlapped parts are less than 20%.
+
+    Args:
+        paragraph (str): The input paragraph.
+        keywords (list[str]): List of important keywords.
+        phrase_length (int): The length of phrases to extract (default is 5 words).
+
+    Returns:
+        list: A list of important phrases.
+    """
+    # Tokenize the paragraph into words
+    words = word_tokenize(paragraph.lower())
+    
+    # Determine phrase length (between 3 and 7 words)
+    phrase_length = min(max(len(words) // 10, 5), 7)
+    
+    # Generate n-grams (phrases) from the paragraph
+    phrases = list(ngrams(words, phrase_length))
+    
+    important_phrases = []
+    used_indices = set()
+    
+    for i, phrase in enumerate(phrases):
+        # Check if the phrase contains any keyword
+        if any(keyword.lower() in phrase for keyword in keywords):
+            # Check overlap with previously selected phrases
+            if not any(abs(i - j) < phrase_length * 0.8 for j in used_indices):
+                important_phrases.append(clean_text(" ".join(phrase)))
+                used_indices.add(i)
+    
+    return important_phrases
+
+"""# Example usage
+keywords = get_keywords(paragraph)
+important_phrases = extract_important_phrases(paragraph, keywords)
+
+print("# Important phrases:")
+for i, phrase in enumerate(important_phrases[:5], 1):  # Print top 5 phrases
+    print(f"{i}. {phrase}")"""

src/application/text/identity.py

@@ -0,0 +1,63 @@
+from transformers import pipeline
+
+ner_pipeline = pipeline("ner")
+
+def extract_entities(text):
+    output = ner_pipeline(text)
+    words = extract_words(output)
+    words = combine_subwords(words)
+    
+    # extract word in each entity and assign to a list of entities, connect words if there is no space between them
+    entities = []    
+    for entity in words:
+        if entity not in entities:
+            entities.append(entity)
+    
+    return entities
+
+
+def extract_words(entities):
+    """
+    Extracts the words from a list of entities.
+
+    Args:
+    entities: A list of entities.
+
+    Returns:
+    A list of words extracted from the entities.
+    """
+    words = []
+    for entity in entities:
+        words.append(entity["word"])
+    return words
+
+
+def combine_subwords(word_list):
+    """
+    Combines subwords (indicated by "##") with the preceding word in a list.
+
+    Args:
+    word_list: A list of words, where subwords are prefixed with "##".
+
+    Returns:
+    A new list with subwords combined with their preceding words.
+    """
+    result = []
+    i = 0
+    while i < len(word_list):
+        if word_list[i].startswith("##"):
+            result[-1] += word_list[i][2:]  # Remove "##" and append to the previous word
+        elif i < len(word_list) - 2 and word_list[i + 1] == "-":  # Combine hyphenated words
+            result.append(word_list[i] + word_list[i + 1] + word_list[i + 2])
+            i += 2  # Skip the next two words
+        else:
+            result.append(word_list[i])
+        i += 1
+    return result
+
+if __name__ == "__main__":
+    text = "The Saudi authorities, I am told, are currently working flat out" \
+            "to collate everything they have on the Magdeburg market suspect," \
+            "Taleb al-Abdulmohsen, and to share it with Germany's ongoing" \
+            "investigation"
+    print(extract_entities(text))

src/application/text/model_detection.py

@@ -0,0 +1,47 @@
+from transformers import pipeline
+
+# TODO: move to a config file
+DEFAULT_MODEL = "Hello-SimpleAI/chatgpt-detector-roberta"
+
+MODEL_HUMAN_LABEL = {DEFAULT_MODEL: "Human"}
+HUMAN = "HUMAN"
+MACHINE = "MACHINE"
+UNKNOWN = "UNKNOWN"
+PARAPHRASE = "PARAPHRASE"
+NON_PARAPHRASE = "NON_PARAPHRASE"
+
+
+def detect_by_ai_model(
+    input_text: str,
+    model: str = DEFAULT_MODEL,
+    max_length: int = 512,
+) -> tuple:
+    """
+    Model: chatgpt_detector_roberta
+    Ref: https://huggingface.co/Hello-SimpleAI/chatgpt-detector-roberta
+    
+    Detects if text is human or machine generated.
+
+    Returns:
+        tuple: (label, confidence_score)
+            where label is HUMAN or MACHINE.
+    """
+    try:
+        pipe = pipeline(
+            "text-classification",
+            model=model,
+            tokenizer=model,
+            max_length=max_length,
+            truncation=True,
+            device_map="auto",  # good for GPU usage
+        )
+        result = pipe(input_text)[0]
+        confidence_score = result["score"]
+        if result["label"] == MODEL_HUMAN_LABEL[model]:
+            label = HUMAN
+        else:
+            label = MACHINE
+        return label, confidence_score
+    except Exception as e:  # Add exception handling
+        print(f"Error in Roberta model inference: {e}")
+        return UNKNOWN, 0.0  # Return UNKNOWN and 0.0 confidence if error

src/application/text/preprocessing.py

@@ -0,0 +1,22 @@
+from nltk.tokenize import sent_tokenize
+
+def split_into_sentences(input_text):
+    """
+    Splits input text into sentences by newlines.
+
+    Args:
+        input_text: The input text as a string.
+
+    Returns:
+        A list of sentences. Returns an empty list if input is not valid.
+    """
+    if not isinstance(input_text, str):
+        return []
+
+    paragraphs = input_text.splitlines()
+    sentences = []
+    for paragraph in paragraphs:
+        paragraph = paragraph.strip()
+        if paragraph:
+            sentences.extend(sent_tokenize(paragraph))
+    return sentences

src/application/text/search.py

@@ -0,0 +1,171 @@
+from collections import Counter
+import os
+import string
+import requests
+from dotenv import load_dotenv
+from nltk.corpus import stopwords
+from nltk.tokenize import sent_tokenize, word_tokenize
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from src.application.text.identity import extract_entities
+
+load_dotenv() 
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+SEARCH_ENGINE_ID = os.getenv("SEARCH_ENGINE_ID")
+
+def search_by_google(
+    query, 
+    num_results=10,
+    is_exact_terms = False
+    ) -> dict:
+    """
+    Searches the Google Custom Search Engine for the given query.
+
+    Args:
+        query: The search query.
+        is_exact_terms: Whether to use exact terms search (True) or regular search (False).
+        num_results: The number of results to return (default: 10).
+
+    Returns:
+        A dictionary containing the search results or None if there was an error.
+    """
+    
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {
+        "key": GOOGLE_API_KEY,
+        "cx": SEARCH_ENGINE_ID,
+        "num": num_results,
+    }
+    if is_exact_terms:
+        params["exactTerms"] = query
+    else:
+        params["q"] = query.replace('"', "")
+    
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        print(f"Error: {response.status_code}, {response.text}")
+        return None
+
+def get_most_frequent_words(input_text, number_word=32):
+    """
+    Gets the top words from the input text, excluding stop words and punctuation.
+
+    Args:
+        input_text: The input text as a string.
+        number_word: The number of top words to return.
+
+    Returns:
+        A list of tuples, where each tuple contains a word and its frequency.
+        Returns an empty list if input is not a string or is empty.
+    """
+    if not isinstance(input_text, str) or not input_text:
+        return []
+
+    words = word_tokenize(input_text.lower())  # Tokenize and lowercase
+
+    stop_words = set(stopwords.words('english'))
+    punctuation = set(string.punctuation) # get all punctuation
+    filtered_words = [
+        word for word in words
+        if word.isalnum() and word not in stop_words and word not in punctuation
+    ]
+    word_frequencies = Counter(filtered_words)
+    top_words = word_frequencies.most_common(number_word)
+    
+    for top_word in top_words:
+        words.append(top_word[0])
+    
+    if len(words) > 32:
+        search_phrase = " ".join(words[:32])
+    else:
+        search_phrase = " ".join(words[:number_word])
+
+    return search_phrase
+
+def get_chunk(input_text, chunk_length=32, num_chunk=3):
+    """
+    Splits the input text into chunks of a specified length.
+
+    Args:
+        input_text: The input text as a string.
+        num_chunk: The maximum number of chunks to create.
+        chunk_length: The desired length of each chunk (in words).
+
+    Returns:
+        A list of string chunks. 
+        Returns an empty list if input is invalid.
+    """
+    if not isinstance(input_text, str):
+        return []
+
+    chunks = []
+    input_words = input_text.split()  # Split by any whitespace
+
+    for i in range(num_chunk):
+        start_index = i * chunk_length
+        end_index = (i + 1) * chunk_length
+        chunk = " ".join(input_words[start_index:end_index])
+        if chunk:  # Only append non-empty chunks
+            chunks.append(chunk)
+
+    return chunks
+
+def get_keywords(text, num_keywords=5):
+    """Return top k keywords from a doc using TF-IDF method"""
+    
+    # Create a TF-IDF Vectorizer
+    vectorizer = TfidfVectorizer(stop_words='english')
+    
+    # Fit and transform the text
+    tfidf_matrix = vectorizer.fit_transform([text])
+    
+    # Get feature names (words)
+    feature_names = vectorizer.get_feature_names_out()
+    
+    # Get TF-IDF scores
+    tfidf_scores = tfidf_matrix.toarray()[0]
+    
+    # Sort words by TF-IDF score
+    word_scores = list(zip(feature_names, tfidf_scores))
+    word_scores.sort(key=lambda x: x[1], reverse=True)
+    
+    # Return top keywords
+    return [word for word, score in word_scores[:num_keywords]]
+
+
+def generate_search_phrases(input_text):
+    """
+    Generates different types of phrases for search purposes.
+
+    Args:
+        input_text: The input text.
+
+    Returns:
+        A list containing:
+        - A list of most frequent words.
+        - The original input text.
+        - A list of text chunks.
+    """
+    if not isinstance(input_text, str):
+        return []
+    
+    search_phrases = []
+    
+    # Method 1: Get most frequent words
+    search_phrases.append(get_most_frequent_words(input_text))
+    
+    # Method 2: Get the whole text
+    search_phrases.append(input_text)
+    
+    # Method 3: Split text by chunks
+    search_phrases.extend(get_chunk(input_text))
+    
+    # Method 4: Get most identities and key words
+    entities = extract_entities(input_text)
+    keywords = get_keywords(input_text, 16)
+    search_phrase = " ".join(entities) + " " + " ".join(keywords)
+    search_phrases.append(search_phrase)
+    
+    return search_phrases

src/application/text/search_detection.py

@@ -0,0 +1,284 @@
+import warnings
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+from src.application.text.preprocessing import split_into_sentences
+from src.application.text.search import generate_search_phrases, search_by_google
+from src.application.url_reader import URLReader
+import numpy as np
+import nltk
+import torch
+from nltk.corpus import stopwords
+from sentence_transformers import SentenceTransformer, util
+import math
+
+from difflib import SequenceMatcher
+
+# Download necessary NLTK data files
+nltk.download('punkt', quiet=True)
+nltk.download('punkt_tab', quiet=True)
+nltk.download('stopwords', quiet=True)
+
+# load the model
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+PARAPHASE_MODEL = SentenceTransformer('paraphrase-MiniLM-L6-v2')
+PARAPHASE_MODEL.to(DEVICE)
+
+BATCH_SIZE = 8
+
+PARAPHRASE_THRESHOLD = 0.8
+PARAPHRASE_THRESHOLD_FOR_OPPOSITE = 0.7
+MIN_SAME_SENTENCE_LEN = 6
+MIN_PHRASE_SENTENCE_LEN = 10
+MIN_RATIO_PARAPHRASE_NUM = 0.7
+MAX_CHAR_SIZE = 30000
+
+
+def detect_by_relative_search(input_text, is_support_opposite = False):
+
+    checked_urls = set()
+    searched_phrases = generate_search_phrases(input_text)
+
+    for candidate in searched_phrases:
+        search_results = search_by_google(candidate)
+        urls = [item['link'] for item in search_results.get("items", [])]
+
+        for url in urls[:3]:
+            if url in checked_urls: # visited url
+                continue
+            checked_urls.add(url)
+            print(f"\tChecking URL: {url}")
+            
+            content = URLReader(url)
+            
+            if content.is_extracted is True:
+                page_text = content.title + "\n" + content.text
+                if page_text is None or len(page_text) > MAX_CHAR_SIZE:
+                    print(f"\t\t↑↑↑ More than {MAX_CHAR_SIZE} characters")
+                    continue
+                is_paraphrase, aligned_sentences = check_paraphrase(input_text, page_text)
+                if is_paraphrase:
+                    return is_paraphrase, url, aligned_sentences
+    return False, None, []
+
+def longest_common_subsequence(arr1, arr2):
+    """
+    Finds the length of the longest common subsequence (contiguous) between
+        two arrays.
+
+    Args:
+        arr1: The first array.
+        arr2: The second array.
+
+    Returns:
+        The length of the longest common subsequence. 
+        Returns 0 if either input is invalid.
+    """
+
+    if not isinstance(arr1, list) or not isinstance(arr2, list):
+        return 0
+
+    n = len(arr1)
+    m = len(arr2)
+
+    if n == 0 or m == 0: #handle empty list
+        return 0
+
+    # Create table dp with size (n+1) x (m+1)
+    dp = [[0] * (m + 1) for _ in range(n + 1)]
+    max_length = 0
+
+    for i in range(1, n + 1):
+        for j in range(1, m + 1):
+            if arr1[i - 1] == arr2[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1] + 1
+                max_length = max(max_length, dp[i][j])
+            else:
+                dp[i][j] = 0  # set 0 since the array must be consecutive
+
+    return max_length
+
+
+def check_sentence(input_sentence, source_sentence, min_same_sentence_len,
+                   min_phrase_sentence_len, verbose=False):
+    """
+    Checks if two sentences are similar based on exact match or 
+        longest common subsequence.
+
+    Args:
+        input_sentence: The input sentence.
+        source_sentence: The source sentence.
+        min_same_sentence_len: Minimum length for exact sentence match.
+        min_phrase_sentence_len: Minimum length for common subsequence match.
+        verbose: If True, print debug information.
+
+    Returns:
+        True if the sentences are considered similar, False otherwise.
+        Returns False if input is not valid.
+    """
+
+    if not isinstance(input_sentence, str) or not isinstance(source_sentence, str):
+        return False
+
+    input_sentence = input_sentence.strip()
+    source_sentence = source_sentence.strip()
+
+    if not input_sentence or not source_sentence:  # handle empty string
+        return False
+
+    input_words = input_sentence.split()  # split without arguments
+    source_words = source_sentence.split()  # split without arguments
+
+    if input_sentence == source_sentence and len(input_words) >= min_same_sentence_len:
+        if verbose:
+            print("Exact match found.")
+        return True
+
+    max_overlap_len = longest_common_subsequence(input_words, source_words)
+    if verbose:
+        print(f"Max overlap length: {max_overlap_len}")  # print overlap length
+    if max_overlap_len >= min_phrase_sentence_len:
+        return True
+
+    return False
+
+
+def check_paraphrase(input_text, page_text, verbose=False):
+    """
+    Checks if the input text is paraphrased in the content at the given URL.
+
+    Args:
+        input_text: The text to check for paraphrase.
+        url: The URL of the web page to compare with.
+        verbose: If True, print debug information.
+
+    Returns:
+        A tuple containing:
+            - is_paraphrase: True if the input text is considered a paraphrase, False otherwise.
+            - paraphrase_results: A list of dictionaries, each containing:
+                - input_sentence: The sentence from the input text.
+                - matched_sentence: The corresponding sentence from the web page (if found).
+                - similarity: The cosine similarity score between the sentences.
+                - is_paraphrase_sentence: True if the individual sentence pair meets the paraphrase criteria, False otherwise.
+    """
+    is_paraphrase_text = False
+    
+    if not isinstance(input_text, str) or not isinstance(page_text, str):
+        return False, []
+
+    # Extract sentences from input text and web page
+    #input_text = remove_punctuation(input_text)
+    input_sentences = split_into_sentences(input_text)
+    
+
+    if not page_text:
+        return is_paraphrase_text, []
+    #page_text = remove_punctuation(page_text)
+    page_sentences = split_into_sentences(page_text)
+
+    if not input_sentences or not page_sentences:
+        return is_paraphrase_text, []
+
+    additional_sentences = []
+    for sentence in page_sentences:
+        if ", external" in sentence:
+            additional_sentences.append(sentence.replace(", external", ""))
+    page_sentences.extend(additional_sentences)
+    
+    min_matching_sentences = math.ceil(len(input_sentences) * MIN_RATIO_PARAPHRASE_NUM)
+
+    # Encode sentences into embeddings
+    embeddings1 = PARAPHASE_MODEL.encode(input_sentences, convert_to_tensor=True, device=DEVICE)
+    embeddings2 = PARAPHASE_MODEL.encode(page_sentences, convert_to_tensor=True, device=DEVICE)
+
+    # Compute cosine similarity matrix
+    similarity_matrix = util.cos_sim(embeddings1, embeddings2).cpu().numpy()
+
+    # Find sentence alignments
+    alignment = []
+    paraphrased_sentence_count = 0
+    for i, sentence1 in enumerate(input_sentences):
+        max_sim_index = np.argmax(similarity_matrix[i])
+        max_similarity = similarity_matrix[i][max_sim_index]
+
+        is_paraphrase_sentence = max_similarity > PARAPHRASE_THRESHOLD
+
+        if 0.80 < max_similarity < 0.99:
+            print(f"\t\tinput_sentence  : {sentence1}")
+            print(f"\t\tmatched_sentence: {page_sentences[max_sim_index]}")
+            print(f"\t\t-->   similarity: {max_similarity}\n")
+        item = {
+            "input_sentence": sentence1,
+            "matched_sentence": page_sentences[max_sim_index],
+            "similarity": max_similarity,
+            "is_paraphrase_sentence": is_paraphrase_sentence,
+        }
+
+        # Check for individual sentence paraphrase if overall paraphrase not yet found
+        if not is_paraphrase_text and check_sentence(
+            sentence1, page_sentences[max_sim_index], MIN_SAME_SENTENCE_LEN, MIN_PHRASE_SENTENCE_LEN
+        ):
+            is_paraphrase_text = True
+            if verbose:
+                print(f"Paraphrase found for individual sentence: {sentence1}")
+                print(f"Matched sentence: {page_sentences[max_sim_index]}")
+
+        alignment.append(item)
+        paraphrased_sentence_count += 1 if is_paraphrase_sentence else 0
+
+    # Check if enough sentences are paraphrases
+    
+    is_paraphrase_text = paraphrased_sentence_count >= min_matching_sentences
+
+    if verbose:
+        print (f"\t\tparaphrased_sentence_count: {paraphrased_sentence_count}, min_matching_sentences: {min_matching_sentences}, total_sentence_count: {len(input_sentences)}")
+        print(f"Minimum matching sentences required: {min_matching_sentences}")
+        print(f"Total input sentences: {len(input_sentences)}")
+        print(f"Number of matching sentences: {paraphrased_sentence_count}")
+        print(f"Is paraphrase: {is_paraphrase_text}")
+        for item in alignment:
+            print(item)
+
+    return is_paraphrase_text, alignment
+
+def similarity_ratio(a, b):
+    """
+    Calculates the similarity ratio between two strings using SequenceMatcher.
+
+    Args:
+        a: The first string.
+        b: The second string.
+
+    Returns:
+        A float representing the similarity ratio between 0.0 and 1.0.
+        Returns 0.0 if either input is None or not a string.
+    """
+    if not isinstance(a, str) or not isinstance(b, str) or a is None or b is None:
+        return 0.0  # Handle cases where inputs are not strings or None
+    return SequenceMatcher(None, a, b).ratio()
+
+def check_human(data, min_ratio=MIN_RATIO_PARAPHRASE_NUM):
+    """
+    Checks if a sufficient number of input sentences are found within
+        source sentences.
+
+    Returns:
+        bool: True if the condition is met, False otherwise.
+    """
+    if not data:  # Handle empty data case
+        return False
+    min_matching = math.ceil(len(data) * min_ratio)
+
+    count = 0
+    
+    #for input_sentence, source_sentence, similiarity, is_paraprhase in data:
+    for sentence in data:
+        if sentence["similarity"] >= 0.99:
+            count += 1
+    print(f"\tmatching_sentence_count   : {count}, min_matching: {min_matching}")
+    if count >= min_matching:
+        return True
+    return False
+
+
+if __name__ == '__main__':    
+    pass

src/application/url_reader.py

@@ -3,6 +3,8 @@ from bs4 import BeautifulSoup
 from newspaper import article, ArticleException, ArticleBinaryDataException
 import requests
 
+# TODO: move this to a config file
+MAX_URL_SIZE = 2000000 # ~2MB
 class URLReader():
     def __init__(self, url: string, newspaper: bool=True):
         self.url = url
@@ -10,6 +12,14 @@ class URLReader():
         self.title = None  # string
         self.images = None  # list of Image objects
         self.top_image = None  # Image object
+        self.is_extracted = False
+        
+        url_size = self.get_size()
+        if url_size == None or url_size > MAX_URL_SIZE:
+            return
+        else: 
+            self.is_extracted = True
+        
         self.newspaper = newspaper  # True if using newspaper4k, False if using BS
         if self.newspaper is True:
             self.extract_content_newspaper()
@@ -73,4 +83,30 @@ class URLReader():
         paragraphs = soup.find_all('p')
         text = ' '.join([p.get_text() for p in paragraphs])
 
-        self.text = text
+        self.text = text
+        
+    def get_size(self):
+        """
+        Retrieves the size of a URL's content using a HEAD request.
+
+        Args:
+            url: The URL to check.
+
+        Returns:
+            The size of the content in bytes, or None if the size cannot be determined
+            (e.g., due to network errors or missing Content-Length header).
+        """
+        try:
+            response = requests.head(self.url, allow_redirects=True, timeout=5) # Add timeout
+            response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
+
+            content_length = response.headers.get('Content-Length')
+            if content_length is not None:
+                return int(content_length)
+            else:
+                print(f"\t\t↑↑↑ Content-Length header not found")
+                return None
+
+        except requests.exceptions.RequestException as e:
+            print(f"\t\t↑↑↑ Error getting URL size: {e}")
+        return None

src/texts/Search_Text/evaluation.py

@@ -40,15 +40,8 @@ def evaluation(texts):
     results = []
     index = 0
     for text in texts:
-<<<<<<< HEAD
-        print("-" * 50)
-        print(f"index = {index}\t {text[:100]}")
-        bbc = [22, 32, 39, 43, 44, 64, 97]
-        if index not in bbc:
-=======
         if index <= 82:
             print(f"index = {index}")
->>>>>>> 59d2492d76034c795f0dbf2632f17d366fb31f14
             index += 1
             continue