grouping input news

application.py

@@ -124,13 +124,13 @@ with gr.Blocks() as demo:
     #url_input.change(load_image, inputs=url_input, outputs=image_view) 
     
     try:
-        with open('example_text_real.txt','r', encoding='utf-8') as file:
+        with open('examples/example_text_real.txt','r', encoding='utf-8') as file:
             text_real_1 = file.read()
-        with open('example_text_real_2.txt','r', encoding='utf-8') as file:
+        with open('examples/example_text_real_2.txt','r', encoding='utf-8') as file:
             text_real_2 = file.read()
-        with open('example_text_LLM_topic.txt','r', encoding='utf-8') as file:
+        with open('examples/example_text_LLM_topic.txt','r', encoding='utf-8') as file:
             text_llm_topic = file.read()
-        with open('example_text_LLM_modification.txt','r', encoding='utf-8') as file:
+        with open('examples/example_text_LLM_modification.txt','r', encoding='utf-8') as file:
             text_llm_modification = file.read()
     except FileNotFoundError:
         print("File not found.")
@@ -140,9 +140,9 @@ with gr.Blocks() as demo:
     title_1 = "Southampton news: Leeds target striker Cameron Archer"
     title_2 = "Southampton news: Leeds target striker Cameron Archer"
     
-    image_1 = "example_image_real_1.jpg.webp"
-    image_2 = "example_image_real_2.jpg.webp"
-    image_3 = "example_image_real_3.jpg"
+    image_1 = "examples/example_image_real_1.jpg.webp"
+    image_2 = "examples/example_image_real_2.jpg.webp"
+    image_3 = "examples/example_image_real_3.jpg"
     
     gr.Examples(
         examples=[
@@ -159,8 +159,4 @@ with gr.Blocks() as demo:
         ],
     )
 
-demo.launch(share=False)
-
-
-# https://www.bbc.com/travel/article/20250127-one-of-the-last-traders-on-the-silk-road
-# https://bbc.com/future/article/20250110-how-often-you-should-wash-your-towels-according-to-science
+demo.launch(share=False)

application_2.py

@@ -91,7 +91,7 @@ with gr.Blocks() as demo:
         with gr.Column(scale=2):
             with gr.Accordion("News Analysis"):
                 detection_button = gr.Button("Verify news")
-                detailed_analysis = gr.HTML()
+                detailed_analysis = gr.HTML("<br>"*40)
 
     # Connect events
     load_button.click(
@@ -116,36 +116,39 @@ with gr.Blocks() as demo:
     #url_input.change(load_image, inputs=url_input, outputs=image_view) 
     
     try:
-        with open('sample_1.txt','r', encoding='utf-8') as file:
-            text_sample_1 = file.read()
-        with open('sample_2.txt','r', encoding='utf-8') as file:
-            text_sample_2 = file.read()
-        with open('sample_3.txt','r', encoding='utf-8') as file:
-            text_sample_3 = file.read()
+        with open('examples/example_text_real.txt','r', encoding='utf-8') as file:
+            text_real_1 = file.read()
+        with open('examples/example_text_real_2.txt','r', encoding='utf-8') as file:
+            text_real_2 = file.read()
+        with open('examples/example_text_LLM_topic.txt','r', encoding='utf-8') as file:
+            text_llm_topic = file.read()
+        with open('examples/example_text_LLM_modification.txt','r', encoding='utf-8') as file:
+            text_llm_modification = file.read()
     except FileNotFoundError:
         print("File not found.")
     except Exception as e:
         print(f"An error occurred: {e}")
     
-    title_1 = "The ancient discovery that put a Silk Road city back on the map"
-    title_2 = "The modern rediscovery that erased a Silk Road city from the map"
+    title_1 = "Southampton news: Leeds target striker Cameron Archer"
+    title_2 = "Southampton news: Leeds target striker Cameron Archer"
     
-    image_1 = "sample_1.jpg.webp"
-    image_2 = "sample_2.jpg.webp"
+    image_1 = "examples/example_image_real_1.jpg.webp"
+    image_2 = "examples/example_image_real_2.jpg.webp"
+    image_3 = "examples/example_image_real_3.jpg"
     
     gr.Examples(
         examples=[
-            [title_1, image_1, text_sample_1],
-            [title_2, image_2, text_sample_2],
-            [title_1, image_2, text_sample_3],
+            [title_1, image_1, text_real_1 + '\n\n' + text_real_2],
+            [title_1, image_2, text_real_1 + '\n\n' + text_llm_modification],
+            [title_1, image_3, text_real_1 + '\n\n' + text_llm_topic],
         ],
         inputs=[news_title, news_image, news_content],
         label="Examples",
         example_labels=[
             "2 real news",
-            "2 modified news",
-            "1 real news & 1 fake news",
+            "1 real news + 1 LLM modification-based news",
+            "1 real news + 1 LLM topic-based news",
         ],
     )
 
-demo.launch(share=False)
+demo.launch(share=True)

src/application/content_detection.py

@@ -23,7 +23,7 @@ class NewsVerification():
         self.news_prediction_label = ""
         self.news_prediction_score = -1
         
-        self.found_img_url:list[str] = []
+        self.found_img_url:list[str] = ["https://ichef.bbci.co.uk/ace/standard/819/cpsprodpb/8acc/live/86282470-defb-11ef-ba00-65100a906e68.jpg"]
         self.aligned_sentences:list[dict] = []
         self.is_paraphrased:list[bool] = []
         self.analyzed_table:list[list] = []
@@ -50,42 +50,61 @@ class NewsVerification():
         print("\tFrom search engine:")
         # Classify by search engine
         input_sentences = split_into_sentences(self.news_text)
-        for sentence in input_sentences:
-            paraphrase, text_url, aligned_sentence, img_urls = detect_text_by_relative_search(sentence)
-        
-            text_prediction_label = "UNKNOWN"
+        current_index = 0
+        previous_paraphrase = None
+        ai_sentence = {
+            "input_sentence": "",
+            "matched_sentence": "",
+            "label": "",
+            "similarity": None,
+            "paraphrase": False,
+            "url": "",
+            }
+        for index, sentence in enumerate(input_sentences):
+            if current_index >= index:
+                continue
+            print(f"-------index = {index}-------")
+            paraphrase, text_url, searched_sentences, img_urls, current_index = detect_text_by_relative_search(input_sentences, index)             
             if paraphrase is False:
-                # Classify text by AI model
-                print("\tFrom AI model:")
-                text_prediction_label, text_prediction_score = detect_text_by_ai_model(sentence)
-                if aligned_sentence == []:
-                    aligned_sentence = {
-                        "input_sentence": sentence,
-                        "matched_sentence": "",
-                        "similarity": text_prediction_score,
-                        "is_paraphrase_sentence": False,
-                        "url": "",
-                    }
+                # add sentence to ai_sentence
+                ai_sentence["input_sentence"] += sentence
+                if index == len(input_sentences) - 1:
+                    # add ai_sentences to align_sentences
+                    text_prediction_label, text_prediction_score = detect_text_by_ai_model(ai_sentence["input_sentence"])
+                    ai_sentence["label"] = text_prediction_label
+                    ai_sentence["similarity"] = text_prediction_score
+                    self.aligned_sentences.append(ai_sentence)
             else:
-                self.found_img_url.extend(img_urls)  # TODO: for demo purposes
-                self.found_img_url.append(img_urls[0])  # TODO: for demo purposes
-                text_prediction_score = aligned_sentence["similarity"]
-                if check_human(aligned_sentence):
-                    text_prediction_label = "HUMAN"
-                else:
-                    text_prediction_label = "MACHINE"
-            
-            print(f"\ttext_prediction_label: {text_prediction_label}\n")
-            self.text_prediction_label.append(text_prediction_label)
-            self.aligned_sentences.append(aligned_sentence)
-            self.is_paraphrased.append(paraphrase)
-            self.text_referent_url.append(text_url)
-            self.text_prediction_score.append(text_prediction_score)
-            paraphrase = False
-            text_url = ""
-            aligned_sentence = {}
-            img_urls = []
-        self.found_img_url = list(set(self.found_img_url))
+                if previous_paraphrase is False or previous_paraphrase is None:
+                    # add ai_sentences to align_sentences
+                    if ai_sentence["input_sentence"] != "":
+                        text_prediction_label, text_prediction_score = detect_text_by_ai_model(ai_sentence["input_sentence"])
+                        ai_sentence["label"] = text_prediction_label
+                        ai_sentence["similarity"] = text_prediction_score
+                        self.aligned_sentences.append(ai_sentence)
+                    
+                        # reset
+                        ai_sentence = {
+                            "input_sentence": "",
+                            "matched_sentence": "",
+                            "label": "",
+                            "similarity": None,
+                            "paraphrase": False,
+                            "url": "",
+                            }
+                    
+                # add searched_sentences to align_sentences
+                if searched_sentences["input_sentence"] != "":
+                    self.found_img_url.extend(img_urls)
+                    if check_human(searched_sentences):
+                        searched_sentences["label"] = "HUMAN"
+                    else:
+                        searched_sentences["label"] = "MACHINE"
+                        
+                    self.aligned_sentences.append(searched_sentences)
+
+            previous_paraphrase = paraphrase
+        #self.found_img_url = list(set(self.found_img_url))
 
     def detect_image_origin(self):
         print("CHECK IMAGE:")
@@ -95,7 +114,8 @@ class NewsVerification():
             self.image_referent_url = None
             return
         
-        print(f"\t: Img path: {self.news_image}")    
+        for image in self.found_img_url:
+            print(f"\tfound_img_url: {image}")    
         matched_url, similarity = detect_image_from_news_image(self.news_image, self.found_img_url)
         if matched_url is not None:
             print(f"matching image: {matched_url}\nsimilarity: {similarity}\n")
@@ -114,6 +134,7 @@ class NewsVerification():
         
         detected_label, score = detect_image_by_ai_model(self.news_image)
         if detected_label:
+            print(f"detected_label: {detected_label} ({score})")
             self.image_prediction_label = detected_label
             self.image_prediction_score = score
             self.image_referent_url = None
@@ -346,13 +367,15 @@ class NewsVerification():
         
         # short_url = self.shorten_url(self.text_referent_url[index], max_length)
         # source_text_url = f"""<a href="{self.text_referent_url[index]}">{short_url}</a>"""
-        
-        self.text_prediction_score[index]
-        return f"""<tr><td>{input_sentence}</td><td>{source_sentence}</td><td>{self.text_prediction_label[index]}<br>({self.text_prediction_score[index]*100:.2f}%)</td><td>{source_text_url}</td></tr>"""
+        #label = self.aligned_sentences[index]["label"]
+        print(self.aligned_sentences)
+        print(index)
+        label = self.aligned_sentences[index]["label"]
+        score = self.aligned_sentences[index]["similarity"]
+        return f"""<tr><td>{input_sentence}</td><td>{source_sentence}</td><td>{label}<br>({score*100:.2f}%)</td><td>{source_text_url}</td></tr>"""
         
     def format_image_row(self, max_length=30):        
-        input_image = f"""<img src="example_image_input.jpg" width="200" height="150">"""
-        print(f"self.news_image = {self.news_image}")
+        # input_image = f"""<img src="example_image_input.jpg" width="200" height="150">"""
         
         if self.image_referent_url is not None or self.image_referent_url != "":
             source_image = f"""<img src="{self.image_referent_url}" width="200" height="150">"""
@@ -360,6 +383,8 @@ class NewsVerification():
             source_image_url = f"""<a href="{self.image_referent_url}">{short_url}</a>"""
         else:
             source_image = "Image not found"
+            source_image_url = ""
+
         return f"""<tr><td>input image</td><td>{source_image}</td><td>{self.image_prediction_label}<br>({self.image_prediction_score:.2f}%)</td><td>{source_image_url}</td></tr>"""
     
     def shorten_url(self, url, max_length=30):

src/application/image/model_detection.py

@@ -130,7 +130,7 @@ def image_generation_detection(image_path):
         image_prediction_label = "MACHINE"
     image_confidence = min(1, 0.5 + abs(prediction - 0.2))
     result += f" with confidence = {round(image_confidence * 100, 2)}%"
-    image_confidence = round(image_confidence * 100, 2)
+    # image_confidence = round(image_confidence * 100, 2)
     return image_prediction_label, image_confidence
 
 

src/application/text/helper.py

@@ -144,6 +144,35 @@ def extract_important_phrases(paragraph: str, keywords: list[str], phrase_length
     
     return important_phrases
 
+def connect_consecutive_indexes(nums):
+    """
+    Connects consecutive integers in a list.
+
+    Args:
+        nums: A list of integers.
+
+    Returns:
+        A list of lists, where each inner list represents a consecutive range.
+    """
+
+    if not nums:  # Handle empty input
+        return []
+
+    result = []
+    start = nums[0]
+    end = nums[0]
+
+    for i in range(1, len(nums)):
+        if nums[i] == end + 1:
+            end = nums[i]
+        else:
+            result.append([start, end])
+            start = nums[i]
+            end = nums[i]
+
+    result.append([start, end])  # Add the last range
+    return result
+
 """# Example usage
 keywords = get_keywords(paragraph)
 important_phrases = extract_important_phrases(paragraph, keywords)

src/application/text/search_detection.py

@@ -33,10 +33,9 @@ MIN_RATIO_PARAPHRASE_NUM = 0.7
 MAX_CHAR_SIZE = 30000
 
 
-def detect_text_by_relative_search(input_text, is_support_opposite = False):
-
+def detect_text_by_relative_search(input_text, index, is_support_opposite = False):
     checked_urls = set()
-    searched_phrases = generate_search_phrases(input_text)
+    searched_phrases = generate_search_phrases(input_text[index])
 
     for candidate in searched_phrases:
         search_results = search_by_google(candidate)
@@ -59,15 +58,36 @@ def detect_text_by_relative_search(input_text, is_support_opposite = False):
                     continue
                 
                 page_text = content.title + "\n" + content.text
+                print(f"page_text: {page_text}")
                 if len(page_text) > MAX_CHAR_SIZE:
                     print(f"\t\t\t↑↑↑ More than {MAX_CHAR_SIZE} characters")
                     continue
                 
-                is_paraphrase, aligned_sentences = check_paraphrase(input_text, page_text, url)
-                #if is_paraphrase:
-                return is_paraphrase, url, aligned_sentences, content.images
+                paraphrase, aligned_first_sentences = check_paraphrase(input_text[index], page_text, url)
+                
+                if paraphrase is False:
+                    return paraphrase, url, aligned_first_sentences, content.images, index
+                
+                sub_paraphrase = True
+                while sub_paraphrase == True:
+                    index += 1
+                    print(f"----search {index}----")
+                    if index >= len(input_text):
+                        break
+                    sub_paraphrase, sub_sentences = check_paraphrase(input_text[index], page_text, url)
+                    print(f"sub_paraphrase: {sub_paraphrase}")
+                    print(f"sub_sentences: {sub_sentences}")
+                    if sub_paraphrase == True:                        
+                        aligned_first_sentences["input_sentence"] += sub_sentences["input_sentence"]
+                        aligned_first_sentences["matched_sentence"] += sub_sentences["matched_sentence"]
+                        aligned_first_sentences["similarity"] += sub_sentences["similarity"]
+                        aligned_first_sentences["similarity"] /= 2
+                
+                print(f"paraphrase: {paraphrase}")
+                print(f"aligned_first_sentences: {aligned_first_sentences}")
+                return paraphrase, url, aligned_first_sentences, content.images, index
                 
-    return False, None, [], []
+    return False, None, [], [], index
 
 def longest_common_subsequence(arr1, arr2):
     """
@@ -151,7 +171,7 @@ def check_sentence(input_sentence, source_sentence, min_same_sentence_len,
     return False
 
 
-def check_paraphrase(input_text, page_text, url, verbose=False):
+def check_paraphrase(input_text, page_text, url):
     """
     Checks if the input text is paraphrased in the content at the given URL.
 
@@ -183,7 +203,6 @@ def check_paraphrase(input_text, page_text, url, verbose=False):
         return is_paraphrase_text, []
     #page_text = remove_punctuation(page_text)
     page_sentences = split_into_sentences(page_text)
-
     if not input_sentences or not page_sentences:
         return is_paraphrase_text, []
 
@@ -193,7 +212,7 @@ def check_paraphrase(input_text, page_text, url, verbose=False):
             additional_sentences.append(sentence.replace(", external", ""))
     page_sentences.extend(additional_sentences)
     
-    min_matching_sentences = math.ceil(len(input_sentences) * MIN_RATIO_PARAPHRASE_NUM)
+    # min_matching_sentences = math.ceil(len(input_sentences) * MIN_RATIO_PARAPHRASE_NUM)
 
     # Encode sentences into embeddings
     embeddings1 = PARAPHASE_MODEL.encode(input_sentences, convert_to_tensor=True, device=DEVICE)
@@ -206,18 +225,18 @@ def check_paraphrase(input_text, page_text, url, verbose=False):
     alignment = {}
     paraphrased_sentence_count = 0
     for i, sentence1 in enumerate(input_sentences):
-        print(f"allign: {i}")
         max_sim_index = np.argmax(similarity_matrix[i])
         max_similarity = similarity_matrix[i][max_sim_index]
 
         is_paraphrase_sentence = max_similarity > PARAPHRASE_THRESHOLD
 
-        if 0.80 > max_similarity:
+        if is_paraphrase_sentence is False:
             alignment = {
                 "input_sentence": sentence1,
                 "matched_sentence": "",
                 "similarity": max_similarity,
-                "is_paraphrase_sentence": is_paraphrase_sentence,
+                "label": "",
+                "paraphrase": is_paraphrase_sentence,
                 "url": "",
             }
         else:
@@ -225,7 +244,8 @@ def check_paraphrase(input_text, page_text, url, verbose=False):
                 "input_sentence": sentence1,
                 "matched_sentence": page_sentences[max_sim_index],
                 "similarity": max_similarity,
-                "is_paraphrase_sentence": is_paraphrase_sentence,
+                "label": "",
+                "paraphrase": is_paraphrase_sentence,
                 "url": url,
             }
 
@@ -234,9 +254,6 @@ def check_paraphrase(input_text, page_text, url, verbose=False):
             sentence1, page_sentences[max_sim_index], MIN_SAME_SENTENCE_LEN, MIN_PHRASE_SENTENCE_LEN
         ):
             is_paraphrase_text = True
-            if verbose:
-                print(f"Paraphrase found for individual sentence: {sentence1}")
-                print(f"Matched sentence: {page_sentences[max_sim_index]}")
 
         #alignment.append(item)
         paraphrased_sentence_count += 1 if is_paraphrase_sentence else 0
@@ -245,15 +262,6 @@ def check_paraphrase(input_text, page_text, url, verbose=False):
     
     is_paraphrase_text = paraphrased_sentence_count > 0 #min_matching_sentences
 
-    if verbose:
-        print (f"\t\tparaphrased_sentence_count: {paraphrased_sentence_count}, min_matching_sentences: {min_matching_sentences}, total_sentence_count: {len(input_sentences)}")
-        print(f"Minimum matching sentences required: {min_matching_sentences}")
-        print(f"Total input sentences: {len(input_sentences)}")
-        print(f"Number of matching sentences: {paraphrased_sentence_count}")
-        print(f"Is paraphrase: {is_paraphrase_text}")
-        for item in alignment:
-            print(item)
-
     return is_paraphrase_text, alignment
 
 def similarity_ratio(a, b):

src/application/url_reader.py

@@ -109,4 +109,11 @@ class URLReader():
 
         except requests.exceptions.RequestException as e:
             print(f"\t\t↑↑↑ Error getting URL size: {e}")
-        return None
+        return None
+    
+
+if __name__ == '__main__':
+    url = "https://www.bbc.com/sport/football/articles/c2d3rdy3673o"
+    reader = URLReader(url)
+    print(f"Title: {reader.title}")
+    print(f"Text: {reader.text}")

src/images/CNN_model_classifier.py

@@ -1,63 +0,0 @@
-import argparse
-
-import torch.nn
-import torchvision.transforms as transforms
-from PIL import Image
-
-from .CNN.networks.resnet import resnet50
-
-
-def predict_cnn(image, model_path, crop=None):
-    model = resnet50(num_classes=1)
-    state_dict = torch.load(model_path, map_location="cpu")
-    model.load_state_dict(state_dict["model"])
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    model.to(device)
-    model.eval()
-
-    # Transform
-    if crop is not None:
-        trans_init = [transforms.CenterCrop(crop)]
-        print("Cropping to [%i]" % crop)
-        trans = transforms.Compose(
-            trans_init
-            + [
-                transforms.ToTensor(),
-                transforms.Normalize(
-                    mean=[0.485, 0.456, 0.406],
-                    std=[0.229, 0.224, 0.225],
-                ),
-            ],
-        )
-
-        image = trans(image.convert("RGB"))
-
-    with torch.no_grad():
-        in_tens = image.unsqueeze(0)
-        prob = model(in_tens).sigmoid().item()
-
-    return prob
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    parser.add_argument("-f", "--file", default="examples_realfakedir")
-    parser.add_argument(
-        "-m",
-        "--model_path",
-        type=str,
-        default="weights/blur_jpg_prob0.5.pth",
-    )
-    parser.add_argument(
-        "-c",
-        "--crop",
-        type=int,
-        default=None,
-        help="by default, do not crop. specify crop size",
-    )
-
-    opt = parser.parse_args()
-    prob = predict_cnn(Image.open(opt.file), opt.model_path, crop=opt.crop)
-    print(f"probability of being synthetic: {prob * 100:.2f}%")

src/images/Diffusion/Pipfile

@@ -1,29 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[[source]]
-url = "https://download.pytorch.org/whl/cu121"
-verify_ssl = true
-name = "downloadpytorch"
-
-[packages]
-pandas = "*"
-numpy = "*"
-polars = "*"
-requests = "*"
-img2dataset = "*"
-torch = {version = "==2.1.0", index = "downloadpytorch"}
-torchvision = {version = "==0.16.0", index = "downloadpytorch"}
-lightning = "*"
-webdataset = "*"
-matplotlib = "*"
-invisible-watermark = "*"
-torchdata = "*"
-timm = "*"
-
-[dev-packages]
-
-[requires]
-python_version = "3.11"

src/images/Diffusion/README.md

@@ -1,72 +0,0 @@
-# AI-generated image detection
-
-This is a group project developed by a team of two individuals.
-
-## Managing Python packages
-
-Use of `pipenv` is recommended. The required packages are in `Pipfile`, and can be installed using `pipenv install`.
-
-## Scraping script for Reddit
-
-`python scrape.py --subreddit midjourney --flair Showcase`
-
-This command will scrape the midjourney subreddit, and filter posts that contain the "Showcase" flair. The default number of images to scrape is 30000. The output will contain a parquet file containing metadata, and a csv file containing the urls.
-
-`img2dataset --url_list=urls/midjourney.csv --output_folder=data/midjourney --thread_count=64 --resize_mode=no --output_format=webdataset`
-
-This command will download the images in the webdataset format.
-
-
-## Laion script for real images
-
-`wget -l1 -r --no-parent https://the-eye.eu/public/AI/cah/laion400m-met-release/laion400m-meta/
-mv the-eye.eu/public/AI/cah/laion400m-met-release/laion400m-meta/ .`
-
-This command will download a 50GB url metadata dataset in 32 parquet files.
-
-`sample_laion_script.ipynb`
-
-This script consolidates the parquet files, excludes NSFW images, and selects a subset of 224,917 images.
-
-`combine_laion_script`
-
-This script combines the outputs from earlier into 1 parquet file.
-
-`img2dataset --url_list urls/laion.parquet --input_format "parquet" --url_col "URL" --caption_col "TEXT" --skip_reencode True --output_format webdataset --output_folder data/laion400m_data --processes_count 16 --thread_count 128 --resize_mode no --save_additional_columns '["NSFW","similarity","LICENSE"]' --enable_wandb True`
-
-This command will download the images in the webdataset format.
-
-
-## Data splitting, preprocessing and loading
-
-`data_split.py` splits the data according to 80/10/10. The number of samples:
-
-```
-./data/laion400m_data: (115346, 14418, 14419)
-./data/genai-images/StableDiffusion: (22060, 2757, 2758)
-./data/genai-images/midjourney: (21096, 2637, 2637)
-./data/genai-images/dalle2: (13582, 1697, 1699)
-./data/genai-images/dalle3: (12027, 1503, 1504)
-```
-
-Each sample contains image, target label(1 for GenAI images), and domain label(denoting which generator the image is from). The meaning of the domain label is:
-
-```
-DOMAIN_LABELS = {
-    0: "laion",
-    1: "StableDiffusion",
-    2: "dalle2",
-    3: "dalle3",
-    4: "midjourney"
-}
-```
-
-The `load_dataloader()` function in `dataloader.py` returns a `torchdata.dataloader2.DataLoader2` given a list of domains for GenAI images(subset of `[1, 2, 3, 4]`, LAION will always be included). When building the training dataset, data augmentation and class balanced sampling are applied. It is very memory intensive(>20G) and takes some time to fill its buffer before producing batches. Use the dataloader in this way:
-
-```
-for epoch in range(10):
-    dl.seed(epoch)
-    for d in dl:
-        model(d)
-dl.shutdown()
-```

src/images/Diffusion/combine_laion_script.ipynb

@@ -1,117 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pip install pyspark"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "current_directory = os.getcwd()\n",
-    "print(current_directory)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "os.chdir(current_directory)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "from pyspark.sql import SparkSession\n",
-    "from pyspark.sql.functions import col\n",
-    "\n",
-    "spark = SparkSession.builder.appName(\"CombineParquetFiles\").config(\"spark.executor.memory\", \"8g\").config(\"spark.executor.cores\", \"4\").config(\"spark.executor.instances\", \"3\").config(\"spark.dynamicAllocation.enabled\", \"true\").config(\"spark.task.maxFailures\", 10).getOrCreate()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parquet_directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_sampled'\n",
-    "\n",
-    "output_parquet_file = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_combined'\n",
-    "\n",
-    "df = spark.read.parquet(parquet_directory_path)\n",
-    "\n",
-    "df_coalesced = df.coalesce(1)\n",
-    "\n",
-    "df_coalesced.write.mode('overwrite').parquet(output_parquet_file)\n",
-    "\n",
-    "row_count = df.count()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(row_count)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parquet_directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_combined/part-00000-0190eea7-02ac-4ea0-86fd-0722308c0c58-c000.snappy.parquet'\n",
-    "\n",
-    "df = spark.read.parquet(parquet_directory_path)\n",
-    "\n",
-    "df.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df.count())"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "bloom",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.16"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

src/images/Diffusion/data_split.py

@@ -1,80 +0,0 @@
-import glob
-import json
-
-import webdataset as wds
-
-
-def split_dataset(path, n_train, n_val, n_test, label, domain_label):
-    max_file_size = 1000
-    input_files = glob.glob(path + "/*.tar")
-    src = wds.WebDataset(input_files)
-
-    train_path_prefix = path + "/train"
-    val_path_prefix = path + "/val"
-    test_path_prefix = path + "/test"
-
-    def write_split(dataset, prefix, start, end):
-        n_split = end - start
-        output_files = [
-            f"{prefix}_{i}.tar" for i in range(n_split // max_file_size + 1)
-        ]
-        for i, output_file in enumerate(output_files):
-            print(f"Writing {output_file}")
-            with wds.TarWriter(output_file) as dst:
-                for sample in dataset.slice(
-                    start + i * max_file_size,
-                    min(start + (i + 1) * max_file_size, end),
-                ):
-                    new_sample = {
-                        "__key__": sample["__key__"],
-                        "jpg": sample["jpg"],
-                        "label.cls": label,
-                        "domain_label.cls": domain_label,
-                    }
-                    dst.write(new_sample)
-
-    write_split(src, train_path_prefix, 0, n_train)
-    write_split(src, val_path_prefix, n_train, n_train + n_val)
-    write_split(
-        src,
-        test_path_prefix,
-        n_train + n_val,
-        n_train + n_val + n_test,
-    )
-
-
-def calculate_sizes(path):
-    stat_files = glob.glob(path + "/*_stats.json")
-    total = 0
-    for f in stat_files:
-        with open(f) as stats:
-            total += json.load(stats)["successes"]
-    n_train = int(total * 0.8)
-    n_val = int(total * 0.1)
-    n_test = total - n_train - n_val
-
-    return n_train, n_val, n_test
-
-
-if __name__ == "__main__":
-
-    paths = [
-        "./data/laion400m_data",
-        "./data/genai-images/StableDiffusion",
-        "./data/genai-images/midjourney",
-        "./data/genai-images/dalle2",
-        "./data/genai-images/dalle3",
-    ]
-
-    sizes = []
-    for p in paths:
-        res = calculate_sizes(p)
-        sizes.append(res)
-
-    domain_labels = [0, 1, 4, 2, 3]
-
-    for i, p in enumerate(paths):
-        print(f"{p}: {sizes[i]}")
-        label = 0 if i == 0 else 1
-        print(label, domain_labels[i])
-        split_dataset(p, *calculate_sizes(p), label, domain_labels[i])

src/images/Diffusion/dataloader.py

@@ -1,228 +0,0 @@
-import argparse
-import collections
-import random
-from typing import Iterator
-
-import cv2
-import numpy as np
-import torchdata.datapipes as dp
-from imwatermark import WatermarkEncoder
-from PIL import (
-    Image,
-    ImageFile,
-)
-from torch.utils.data import DataLoader
-from torchdata.datapipes.iter import (
-    Concater,
-    FileLister,
-    FileOpener,
-    SampleMultiplexer,
-)
-from torchvision.transforms import v2
-from tqdm import tqdm
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-Image.MAX_IMAGE_PIXELS = 1000000000
-
-encoder = WatermarkEncoder()
-encoder.set_watermark("bytes", b"test")
-
-
-DOMAIN_LABELS = {
-    0: "laion",
-    1: "StableDiffusion",
-    2: "dalle2",
-    3: "dalle3",
-    4: "midjourney",
-}
-
-N_SAMPLES = {
-    0: (115346, 14418, 14419),
-    1: (22060, 2757, 2758),
-    4: (21096, 2637, 2637),
-    2: (13582, 1697, 1699),
-    3: (12027, 1503, 1504),
-}
-
-
-@dp.functional_datapipe("collect_from_workers")
-class WorkerResultCollector(dp.iter.IterDataPipe):
-    def __init__(self, source: dp.iter.IterDataPipe):
-        self.source = source
-
-    def __iter__(self) -> Iterator:
-        yield from self.source
-
-    def is_replicable(self) -> bool:
-        """Method to force data back to main process"""
-        return False
-
-
-def crop_bottom(image, cutoff=16):
-    return image[:, :-cutoff, :]
-
-
-def random_gaussian_blur(image, p=0.01):
-    if random.random() < p:
-        return v2.functional.gaussian_blur(image, kernel_size=5)
-    return image
-
-
-def random_invisible_watermark(image, p=0.2):
-    image_np = np.array(image)
-    image_np = np.transpose(image_np, (1, 2, 0))
-
-    if image_np.ndim == 2:  # Grayscale image
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)
-    elif image_np.shape[2] == 4:  # RGBA image
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR)
-
-    # print(image_np.shape)
-    if image_np.shape[0] < 256 or image_np.shape[1] < 256:
-        image_np = cv2.resize(
-            image_np,
-            (256, 256),
-            interpolation=cv2.INTER_AREA,
-        )
-    if random.random() < p:
-        return encoder.encode(image_np, method="dwtDct")
-    return image_np
-
-
-def build_transform(split: str):
-    train_transform = v2.Compose(
-        [
-            v2.Lambda(crop_bottom),
-            v2.RandomCrop((256, 256), pad_if_needed=True),
-            v2.Lambda(random_gaussian_blur),
-            v2.RandomGrayscale(p=0.05),
-            v2.Lambda(random_invisible_watermark),
-            v2.ToImage(),
-        ],
-    )
-
-    eval_transform = v2.Compose(
-        [
-            v2.CenterCrop((256, 256)),
-        ],
-    )
-    transform = train_transform if split == "train" else eval_transform
-
-    return transform
-
-
-def dp_to_tuple_train(input_dict):
-    transform = build_transform("train")
-    return (
-        transform(input_dict[".jpg"]),
-        input_dict[".label.cls"],
-        input_dict[".domain_label.cls"],
-    )
-
-
-def dp_to_tuple_eval(input_dict):
-    transform = build_transform("eval")
-    return (
-        transform(input_dict[".jpg"]),
-        input_dict[".label.cls"],
-        input_dict[".domain_label.cls"],
-    )
-
-
-def load_dataset(domains: list[int], split: str):
-
-    laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar")
-    genai_lister = {
-        d: FileLister(
-            f"./data/genai-images/{DOMAIN_LABELS[d]}",
-            f"{split}*.tar",
-        )
-        for d in domains
-        if DOMAIN_LABELS[d] != "laion"
-    }
-    weight_genai = 1 / len(genai_lister)
-
-    def open_lister(lister):
-        opener = FileOpener(lister, mode="b")
-        return opener.load_from_tar().routed_decode().webdataset()
-
-    buffer_size1 = 100 if split == "train" else 10
-    buffer_size2 = 100 if split == "train" else 10
-
-    if split != "train":
-        all_lister = [laion_lister] + list(genai_lister.values())
-        dp = open_lister(Concater(*all_lister)).sharding_filter()
-    else:
-        laion_dp = (
-            open_lister(laion_lister.shuffle())
-            .cycle()
-            .sharding_filter()
-            .shuffle(buffer_size=buffer_size1)
-        )
-        genai_dp = {
-            open_lister(genai_lister[d].shuffle())
-            .cycle()
-            .sharding_filter()
-            .shuffle(buffer_size=buffer_size1): weight_genai
-            for d in domains
-            if DOMAIN_LABELS[d] != "laion"
-        }
-        dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle(
-            buffer_size=buffer_size2,
-        )
-
-    if split == "train":
-        dp = dp.map(dp_to_tuple_train)
-    else:
-        dp = dp.map(dp_to_tuple_eval)
-
-    return dp
-
-
-def load_dataloader(
-    domains: list[int],
-    split: str,
-    batch_size: int = 32,
-    num_workers: int = 4,
-):
-    dp = load_dataset(domains, split)
-    # if split == "train":
-    #     dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42)
-    dp = dp.batch(batch_size).collate()
-    dl = DataLoader(
-        dp,
-        batch_size=None,
-        num_workers=num_workers,
-        pin_memory=True,
-    )
-
-    return dl
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    args = parser.parse_args()
-
-    # testing code
-    dl = load_dataloader([0, 1], "train", num_workers=8)
-    y_dist = collections.Counter()
-    d_dist = collections.Counter()
-
-    for i, (img, y, d) in tqdm(enumerate(dl)):
-        if i % 100 == 0:
-            print(y, d)
-        if i == 400:
-            break
-        y_dist.update(y.numpy())
-        d_dist.update(d.numpy())
-
-    print("class label")
-    for label in sorted(y_dist):
-        frequency = y_dist[label] / sum(y_dist.values())
-        print(f"• {label}: {frequency:.2%} ({y_dist[label]})")
-
-    print("domain label")
-    for label in sorted(d_dist):
-        frequency = d_dist[label] / sum(d_dist.values())
-        print(f"• {label}: {frequency:.2%} ({d_dist[label]})")

src/images/Diffusion/diffusion_data_loader.py

@@ -1,233 +0,0 @@
-import argparse
-import collections
-import glob
-import os
-import random
-from typing import Iterator
-
-import cv2
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-import torchdata as td
-import torchdata.datapipes as dp
-from imwatermark import WatermarkEncoder
-from PIL import (
-    Image,
-    ImageFile,
-)
-from torch.utils.data import (
-    DataLoader,
-    RandomSampler,
-)
-from torchdata.dataloader2 import (
-    DataLoader2,
-    MultiProcessingReadingService,
-)
-from torchdata.datapipes.iter import (
-    Concater,
-    FileLister,
-    FileOpener,
-    SampleMultiplexer,
-)
-from torchvision.transforms import v2
-from tqdm import tqdm
-from utils_sampling import UnderSamplerIterDataPipe
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-Image.MAX_IMAGE_PIXELS = 1000000000
-
-encoder = WatermarkEncoder()
-encoder.set_watermark("bytes", b"test")
-
-DOMAIN_LABELS = {
-    0: "laion",
-    1: "StableDiffusion",
-    2: "dalle2",
-    3: "dalle3",
-    4: "midjourney",
-}
-
-N_SAMPLES = {
-    0: (115346, 14418, 14419),
-    1: (22060, 2757, 2758),
-    4: (21096, 2637, 2637),
-    2: (13582, 1697, 1699),
-    3: (12027, 1503, 1504),
-}
-
-
-@dp.functional_datapipe("collect_from_workers")
-class WorkerResultCollector(dp.iter.IterDataPipe):
-    def __init__(self, source: dp.iter.IterDataPipe):
-        self.source = source
-
-    def __iter__(self) -> Iterator:
-        yield from self.source
-
-    def is_replicable(self) -> bool:
-        """Method to force data back to main process"""
-        return False
-
-
-def crop_bottom(image, cutoff=16):
-    return image[:, :-cutoff, :]
-
-
-def random_gaussian_blur(image, p=0.01):
-    if random.random() < p:
-        return v2.functional.gaussian_blur(image, kernel_size=5)
-    return image
-
-
-def random_invisible_watermark(image, p=0.2):
-    image_np = np.array(image)
-    image_np = np.transpose(image_np, (1, 2, 0))
-
-    if image_np.ndim == 2:  # Grayscale image
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)
-    elif image_np.shape[2] == 4:  # RGBA image
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR)
-
-    # print(image_np.shape)
-    if image_np.shape[0] < 256 or image_np.shape[1] < 256:
-        image_np = cv2.resize(
-            image_np, (256, 256), interpolation=cv2.INTER_AREA
-        )
-    if random.random() < p:
-        return encoder.encode(image_np, method="dwtDct")
-    return image_np
-
-
-def build_transform(split: str):
-    train_transform = v2.Compose(
-        [
-            v2.Lambda(crop_bottom),
-            v2.RandomCrop((256, 256), pad_if_needed=True),
-            v2.Lambda(random_gaussian_blur),
-            v2.RandomGrayscale(p=0.05),
-            v2.Lambda(random_invisible_watermark),
-            v2.ToImage(),
-        ]
-    )
-
-    eval_transform = v2.Compose(
-        [
-            v2.CenterCrop((256, 256)),
-        ]
-    )
-    transform = train_transform if split == "train" else eval_transform
-
-    return transform
-
-
-def dp_to_tuple_train(input_dict):
-    transform = build_transform("train")
-    return (
-        transform(input_dict[".jpg"]),
-        input_dict[".label.cls"],
-        input_dict[".domain_label.cls"],
-    )
-
-
-def dp_to_tuple_eval(input_dict):
-    transform = build_transform("eval")
-    return (
-        transform(input_dict[".jpg"]),
-        input_dict[".label.cls"],
-        input_dict[".domain_label.cls"],
-    )
-
-
-def load_dataset(domains: list[int], split: str):
-    laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar")
-    genai_lister = {
-        d: FileLister(
-            f"./data/genai-images/{DOMAIN_LABELS[d]}", f"{split}*.tar"
-        )
-        for d in domains
-        if DOMAIN_LABELS[d] != "laion"
-    }
-    weight_genai = 1 / len(genai_lister)
-
-    def open_lister(lister):
-        opener = FileOpener(lister, mode="b")
-        return opener.load_from_tar().routed_decode().webdataset()
-
-    buffer_size1 = 100 if split == "train" else 10
-    buffer_size2 = 100 if split == "train" else 10
-
-    if split != "train":
-        all_lister = [laion_lister] + list(genai_lister.values())
-        dp = open_lister(Concater(*all_lister)).sharding_filter()
-    else:
-        laion_dp = (
-            open_lister(laion_lister.shuffle())
-            .cycle()
-            .sharding_filter()
-            .shuffle(buffer_size=buffer_size1)
-        )
-        genai_dp = {
-            open_lister(genai_lister[d].shuffle())
-            .cycle()
-            .sharding_filter()
-            .shuffle(
-                buffer_size=buffer_size1,
-            ): weight_genai
-            for d in domains
-            if DOMAIN_LABELS[d] != "laion"
-        }
-        dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle(
-            buffer_size=buffer_size2
-        )
-
-    if split == "train":
-        dp = dp.map(dp_to_tuple_train)
-    else:
-        dp = dp.map(dp_to_tuple_eval)
-
-    return dp
-
-
-def load_dataloader(
-    domains: list[int], split: str, batch_size: int = 32, num_workers: int = 4
-):
-    dp = load_dataset(domains, split)
-    # if split == "train":
-    #     dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42)
-    dp = dp.batch(batch_size).collate()
-    dl = DataLoader(
-        dp, batch_size=None, num_workers=num_workers, pin_memory=True
-    )
-
-    return dl
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    args = parser.parse_args()
-
-    # testing code
-    dl = load_dataloader([0, 1], "train", num_workers=8)
-    y_dist = collections.Counter()
-    d_dist = collections.Counter()
-
-    for i, (img, y, d) in tqdm(enumerate(dl)):
-        if i % 100 == 0:
-            print(y, d)
-        if i == 400:
-            break
-        y_dist.update(y.numpy())
-        d_dist.update(d.numpy())
-
-    print("class label")
-    for label in sorted(y_dist):
-        frequency = y_dist[label] / sum(y_dist.values())
-        print(f"• {label}: {frequency:.2%} ({y_dist[label]})")
-
-    print("domain label")
-    for label in sorted(d_dist):
-        frequency = d_dist[label] / sum(d_dist.values())
-        print(f"• {label}: {frequency:.2%} ({d_dist[label]})")

src/images/Diffusion/diffusion_model_classifier.py

@@ -1,242 +0,0 @@
-import argparse
-import logging
-import os
-
-import pandas as pd
-import pytorch_lightning as pl
-import timm
-import torch
-import torchvision.transforms as transforms
-from data_split import *
-from dataloader import *
-from PIL import Image
-from pytorch_lightning.callbacks import (
-    EarlyStopping,
-    ModelCheckpoint,
-)
-from sklearn.metrics import roc_auc_score
-from torchmetrics import (
-    Accuracy,
-    Recall,
-)
-from utils_sampling import *
-
-logging.basicConfig(
-    filename="training.log", filemode="w", level=logging.INFO, force=True
-)
-
-
-class ImageClassifier(pl.LightningModule):
-    def __init__(self, lmd=0):
-        super().__init__()
-        self.model = timm.create_model(
-            "resnet50", pretrained=True, num_classes=1
-        )
-        self.accuracy = Accuracy(task="binary", threshold=0.5)
-        self.recall = Recall(task="binary", threshold=0.5)
-        self.validation_outputs = []
-        self.lmd = lmd
-
-    def forward(self, x):
-        return self.model(x)
-
-    def training_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        print(f"Shape of outputs (training): {outputs.shape}")
-        print(f"Shape of labels (training): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        logging.info(f"Training Step - ERM loss: {loss.item()}")
-        loss += self.lmd * (outputs**2).mean()  # SD loss penalty
-        logging.info(f"Training Step - SD loss: {loss.item()}")
-        return loss
-
-    def validation_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        if outputs.shape == torch.Size([]):
-            return
-
-        print(f"Shape of outputs (validation): {outputs.shape}")
-        print(f"Shape of labels (validation): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        preds = torch.sigmoid(outputs)
-        self.log("val_loss", loss, prog_bar=True, sync_dist=True)
-        self.log(
-            "val_acc",
-            self.accuracy(preds, labels.int()),
-            prog_bar=True,
-            sync_dist=True,
-        )
-        self.log(
-            "val_recall",
-            self.recall(preds, labels.int()),
-            prog_bar=True,
-            sync_dist=True,
-        )
-        output = {"val_loss": loss, "preds": preds, "labels": labels}
-        self.validation_outputs.append(output)
-        logging.info(f"Validation Step - Batch loss: {loss.item()}")
-        return output
-
-    def predict_step(self, batch):
-        images, label, domain = batch
-        outputs = self.forward(images).squeeze()
-        preds = torch.sigmoid(outputs)
-        return preds, label, domain
-
-    def on_validation_epoch_end(self):
-        if not self.validation_outputs:
-            logging.warning("No outputs in validation step to process")
-            return
-        preds = torch.cat([x["preds"] for x in self.validation_outputs])
-        labels = torch.cat([x["labels"] for x in self.validation_outputs])
-        if labels.unique().size(0) == 1:
-            logging.warning("Only one class in validation step")
-            return
-        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
-        self.log("val_auc", auc_score, prog_bar=True, sync_dist=True)
-        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
-        self.validation_outputs = []
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
-        return optimizer
-
-
-checkpoint_callback = ModelCheckpoint(
-    monitor="val_loss",
-    dirpath="./model_checkpoints/",
-    filename="image-classifier-{step}-{val_loss:.2f}",
-    save_top_k=3,
-    mode="min",
-    every_n_train_steps=1001,
-    enable_version_counter=True,
-)
-
-early_stop_callback = EarlyStopping(
-    monitor="val_loss",
-    patience=4,
-    mode="min",
-)
-
-
-def load_image(image_path, transform=None):
-    image = Image.open(image_path).convert("RGB")
-
-    if transform:
-        image = transform(image)
-
-    return image
-
-
-def predict_single_image(image_path, model, transform=None):
-    image = load_image(image_path, transform)
-
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-    model.to(device)
-
-    image = image.to(device)
-
-    model.eval()
-
-    with torch.no_grad():
-        image = image.unsqueeze(0)
-        output = model(image).squeeze()
-        print(output)
-        prediction = torch.sigmoid(output).item()
-
-    return prediction
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--ckpt_path", help="checkpoint to continue from", required=False
-)
-parser.add_argument(
-    "--predict", help="predict on test set", action="store_true"
-)
-parser.add_argument("--reset", help="reset training", action="store_true")
-parser.add_argument(
-    "--predict_image",
-    help="predict the class of a single image",
-    action="store_true",
-)
-parser.add_argument(
-    "--image_path",
-    help="path to the image to predict",
-    type=str,
-    required=False,
-)
-args = parser.parse_args()
-
-train_domains = [0, 1, 4]
-val_domains = [0, 1, 4]
-lmd_value = 0
-
-if args.predict:
-    test_dl = load_dataloader(
-        [0, 1, 2, 3, 4], "test", batch_size=128, num_workers=1
-    )
-    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-    trainer = pl.Trainer()
-    predictions = trainer.predict(model, dataloaders=test_dl)
-    preds, labels, domains = zip(*predictions)
-    preds = torch.cat(preds).cpu().numpy()
-    labels = torch.cat(labels).cpu().numpy()
-    domains = torch.cat(domains).cpu().numpy()
-    print(preds.shape, labels.shape, domains.shape)
-    df = pd.DataFrame({"preds": preds, "labels": labels, "domains": domains})
-    filename = "preds-" + args.ckpt_path.split("/")[-1]
-    df.to_csv(f"outputs/{filename}.csv", index=False)
-elif args.predict_image:
-    image_path = args.image_path
-    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-
-    # Define the transformations for the image
-    transform = transforms.Compose(
-        [
-            transforms.Resize((224, 224)),  # Image size expected by ResNet50
-            transforms.ToTensor(),
-            transforms.Normalize(
-                mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
-            ),
-        ]
-    )
-
-    prediction = predict_single_image(image_path, model, transform)
-    print("prediction", prediction)
-
-    # Output the prediction
-    print(
-        f"Prediction for {image_path}: {'Human' if prediction <= 0.001 else 'Generated'}"
-    )
-else:
-    train_dl = load_dataloader(
-        train_domains, "train", batch_size=128, num_workers=4
-    )
-    logging.info("Training dataloader loaded")
-    val_dl = load_dataloader(val_domains, "val", batch_size=128, num_workers=4)
-    logging.info("Validation dataloader loaded")
-
-    if args.reset:
-        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-    else:
-        model = ImageClassifier(lmd=lmd_value)
-    trainer = pl.Trainer(
-        callbacks=[checkpoint_callback, early_stop_callback],
-        max_steps=20000,
-        val_check_interval=1000,
-        check_val_every_n_epoch=None,
-    )
-    trainer.fit(
-        model=model,
-        train_dataloaders=train_dl,
-        val_dataloaders=val_dl,
-        ckpt_path=args.ckpt_path if not args.reset else None,
-    )

src/images/Diffusion/evaluation.ipynb

@@ -1,187 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "import polars as pl\n",
-    "import matplotlib.pyplot as plt\n",
-    "import seaborn as sns\n",
-    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score, RocCurveDisplay\n",
-    "\n",
-    "sns.set()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def pfbeta(labels, predictions, beta=1):\n",
-    "    y_true_count = 0\n",
-    "    ctp = 0\n",
-    "    cfp = 0\n",
-    "\n",
-    "    for idx in range(len(labels)):\n",
-    "        prediction = min(max(predictions[idx], 0), 1)\n",
-    "        if (labels[idx]):\n",
-    "            y_true_count += 1\n",
-    "            ctp += prediction\n",
-    "        else:\n",
-    "            cfp += prediction\n",
-    "\n",
-    "    beta_squared = beta * beta\n",
-    "    c_precision = ctp / (ctp + cfp)\n",
-    "    c_recall = ctp / y_true_count\n",
-    "    if (c_precision > 0 and c_recall > 0):\n",
-    "        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n",
-    "        return result\n",
-    "    else:\n",
-    "        return 0"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_part_metrics(df: pl.DataFrame, threshold=0.3) -> dict:\n",
-    "    df = df.with_columns((df[\"preds\"] > threshold).alias(\"preds_bin\"))\n",
-    "    metrics = {}\n",
-    "    # binary metrics using the threshold\n",
-    "    metrics[\"accuracy\"] = accuracy_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    metrics[\"precision\"] = precision_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    metrics[\"recall\"] = recall_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    metrics[\"f1\"] = f1_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    # probabilistic F1 (doesn't depend on the threshold)\n",
-    "    metrics[\"pf1\"] = pfbeta(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
-    "    # ROC AUC\n",
-    "    metrics[\"roc_auc\"] = roc_auc_score(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
-    "    return metrics\n",
-    "\n",
-    "\n",
-    "def get_all_metrics(df: pl.DataFrame, threshold=0.3) -> pd.DataFrame:\n",
-    "    groups = [list(range(5)), [0, 1], [0, 4], [0, 2], [0, 3]]\n",
-    "    group_names = [\"all\", \"StableDiffusion\", \"Midjourney\", \"Dalle2\", \"Dalle3\"]\n",
-    "    all_metrics = []\n",
-    "    for i, g in enumerate(groups):\n",
-    "        subset = df.filter(pl.col(\"domains\").is_in(g))\n",
-    "        metrics = get_part_metrics(subset, threshold=threshold)\n",
-    "        metrics[\"group\"] = group_names[i]\n",
-    "        all_metrics.append(metrics)\n",
-    "    \n",
-    "    return pd.DataFrame(all_metrics)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df1 = pl.read_csv(\"outputs/preds-image-classifier-1.csv\")\n",
-    "metrics_df1 = get_all_metrics(df1, threshold=0.5)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metrics_df1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df14 = pl.read_csv(\"outputs/preds-image-classifier-14.csv\")\n",
-    "metrics_df14 = get_all_metrics(df14, threshold=0.5)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metrics_df14"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df142 = pl.read_csv(\"outputs/preds-image-classifier-142.csv\")\n",
-    "metrics_df142 = get_all_metrics(df142, threshold=0.5)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metrics_df142"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df1423 = pl.read_csv(\"outputs/preds-image-classifier-1423.csv\")\n",
-    "metrics_df1423 = get_all_metrics(df1423, threshold=0.5)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metrics_df1423"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "GenAI-image-detection-Z_9oJJe7",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

src/images/Diffusion/model.py

@@ -1,307 +0,0 @@
-import argparse
-import logging
-import os
-
-import pandas as pd
-import pytorch_lightning as pl
-import timm
-import torch
-import torch.nn.functional as F
-import torchvision.transforms as transforms
-from dataloader import load_dataloader
-from PIL import Image
-from pytorch_lightning.callbacks import (
-    EarlyStopping,
-    ModelCheckpoint,
-)
-from sklearn.metrics import roc_auc_score
-from torchmetrics import (
-    Accuracy,
-    Recall,
-)
-
-logging.basicConfig(
-    filename="training.log",
-    filemode="w",
-    level=logging.INFO,
-    force=True,
-)
-
-
-class ImageClassifier(pl.LightningModule):
-    def __init__(self, lmd=0):
-        super().__init__()
-        self.model = timm.create_model(
-            "resnet50",
-            pretrained=True,
-            num_classes=1,
-        )
-        self.accuracy = Accuracy(task="binary", threshold=0.5)
-        self.recall = Recall(task="binary", threshold=0.5)
-        self.validation_outputs = []
-        self.lmd = lmd
-
-    def forward(self, x):
-        return self.model(x)
-
-    def training_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        print(f"Shape of outputs (training): {outputs.shape}")
-        print(f"Shape of labels (training): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        logging.info(f"Training Step - ERM loss: {loss.item()}")
-        loss += self.lmd * (outputs**2).mean()  # SD loss penalty
-        logging.info(f"Training Step - SD loss: {loss.item()}")
-        return loss
-
-    def validation_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        if outputs.shape == torch.Size([]):
-            return
-
-        print(f"Shape of outputs (validation): {outputs.shape}")
-        print(f"Shape of labels (validation): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        preds = torch.sigmoid(outputs)
-        self.log("val_loss", loss, prog_bar=True, sync_dist=True)
-        self.log(
-            "val_acc",
-            self.accuracy(preds, labels.int()),
-            prog_bar=True,
-            sync_dist=True,
-        )
-        self.log(
-            "val_recall",
-            self.recall(preds, labels.int()),
-            prog_bar=True,
-            sync_dist=True,
-        )
-        output = {"val_loss": loss, "preds": preds, "labels": labels}
-        self.validation_outputs.append(output)
-        logging.info(f"Validation Step - Batch loss: {loss.item()}")
-        return output
-
-    def predict_step(self, batch):
-        images, label, domain = batch
-        outputs = self.forward(images).squeeze()
-        preds = torch.sigmoid(outputs)
-        return preds, label, domain
-
-    def on_validation_epoch_end(self):
-        if not self.validation_outputs:
-            logging.warning("No outputs in validation step to process")
-            return
-        preds = torch.cat([x["preds"] for x in self.validation_outputs])
-        labels = torch.cat([x["labels"] for x in self.validation_outputs])
-        if labels.unique().size(0) == 1:
-            logging.warning("Only one class in validation step")
-            return
-        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
-        self.log("val_auc", auc_score, prog_bar=True, sync_dist=True)
-        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
-        self.validation_outputs = []
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
-        return optimizer
-
-
-checkpoint_callback = ModelCheckpoint(
-    monitor="val_loss",
-    dirpath="./model_checkpoints/",
-    filename="image-classifier-{step}-{val_loss:.2f}",
-    save_top_k=3,
-    mode="min",
-    every_n_train_steps=1001,
-    enable_version_counter=True,
-)
-
-early_stop_callback = EarlyStopping(
-    monitor="val_loss",
-    patience=4,
-    mode="min",
-)
-
-
-def load_image(image_path, transform=None):
-    image = Image.open(image_path).convert("RGB")
-
-    if transform:
-        image = transform(image)
-
-    return image
-
-
-def predict_single_image(image_path, model, transform=None):
-    image = load_image(image_path, transform)
-
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-    model.to(device)
-
-    image = image.to(device)
-
-    model.eval()
-
-    with torch.no_grad():
-        image = image.unsqueeze(0)
-        output = model(image).squeeze()
-        print(output)
-        prediction = torch.sigmoid(output).item()
-
-    return prediction
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--ckpt_path",
-    help="checkpoint to continue from",
-    required=False,
-)
-parser.add_argument(
-    "--predict",
-    help="predict on test set",
-    action="store_true",
-)
-parser.add_argument("--reset", help="reset training", action="store_true")
-parser.add_argument(
-    "--predict_image",
-    help="predict the class of a single image",
-    action="store_true",
-)
-parser.add_argument(
-    "--image_path",
-    help="path to the image to predict",
-    type=str,
-    required=False,
-)
-parser.add_argument(
-    "--dir",
-    help="path to the images to predict",
-    type=str,
-    required=False,
-)
-parser.add_argument(
-    "--output_file",
-    help="path to output file",
-    type=str,
-    required=False,
-)
-args = parser.parse_args()
-
-train_domains = [0, 1, 4]
-val_domains = [0, 1, 4]
-lmd_value = 0
-
-if args.predict:
-    test_dl = load_dataloader(
-        [0, 1, 2, 3, 4],
-        "test",
-        batch_size=128,
-        num_workers=1,
-    )
-    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-    trainer = pl.Trainer()
-    predictions = trainer.predict(model, dataloaders=test_dl)
-    preds, labels, domains = zip(*predictions)
-    preds = torch.cat(preds).cpu().numpy()
-    labels = torch.cat(labels).cpu().numpy()
-    domains = torch.cat(domains).cpu().numpy()
-    print(preds.shape, labels.shape, domains.shape)
-    df = pd.DataFrame({"preds": preds, "labels": labels, "domains": domains})
-    filename = "preds-" + args.ckpt_path.split("/")[-1]
-    df.to_csv(f"outputs/{filename}.csv", index=False)
-elif args.predict_image:
-    image_path = args.image_path
-    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-
-    # Define the transformations for the image
-    # transform = transforms.Compose(
-    #     [
-    #         transforms.Resize((224, 224)),  # Image size expected by ResNet50
-    #         transforms.ToTensor(),
-    #         transforms.Normalize(
-    #             mean=[0.485, 0.456, 0.406],
-    #             std=[0.229, 0.224, 0.225],
-    #         ),
-    #     ],
-    # )
-
-    transform = transforms.Compose(
-        [
-            transforms.CenterCrop((256, 256)),
-            transforms.ToTensor(),
-        ],
-    )
-
-    prediction = predict_single_image(image_path, model, transform)
-    print("prediction", prediction)
-
-    # Output the prediction
-    print(
-        f"Prediction for {image_path}: "
-        f"{'Human' if prediction <= 0.001 else 'Generated'}",
-    )
-elif args.dir is not None:
-    predictions = []
-    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-    # Define the transformations for the image
-    # transform = transforms.Compose(
-    #     [
-    #         transforms.Resize((224, 224)),  # Image size expected by ResNet50
-    #         transforms.ToTensor(),
-    #         transforms.Normalize(
-    #             mean=[0.485, 0.456, 0.406],
-    #             std=[0.229, 0.224, 0.225],
-    #         ),
-    #     ],
-    # )
-    transform = transforms.Compose(
-        [
-            transforms.CenterCrop((256, 256)),
-            transforms.ToTensor(),
-        ],
-    )
-    for root, dirs, files in os.walk(os.path.abspath(args.dir)):
-        for f_name in files:
-            f = os.path.join(root, f_name)
-            print(f"Predicting: {f}")
-            p = predict_single_image(f, model, transform)
-            predictions.append([f, f.split("/")[-2], p, p > 0.5])
-            print(f"--predicted: {p}")
-
-    df = pd.DataFrame(predictions, columns=["path", "folder", "pred", "class"])
-    df.to_csv(args.output_file, index=False)
-else:
-    train_dl = load_dataloader(
-        train_domains,
-        "train",
-        batch_size=128,
-        num_workers=4,
-    )
-    logging.info("Training dataloader loaded")
-    val_dl = load_dataloader(val_domains, "val", batch_size=128, num_workers=4)
-    logging.info("Validation dataloader loaded")
-
-    if args.reset:
-        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-    else:
-        model = ImageClassifier(lmd=lmd_value)
-    trainer = pl.Trainer(
-        callbacks=[checkpoint_callback, early_stop_callback],
-        max_steps=20000,
-        val_check_interval=1000,
-        check_val_every_n_epoch=None,
-    )
-    trainer.fit(
-        model=model,
-        train_dataloaders=train_dl,
-        val_dataloaders=val_dl,
-        ckpt_path=args.ckpt_path if not args.reset else None,
-    )

src/images/Diffusion/sample_laion_script.ipynb

@@ -1,73 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import dask.dataframe as dd\n",
-    "from dask.diagnostics import ProgressBar\n",
-    "import os\n",
-    "\n",
-    "directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/'\n",
-    "\n",
-    "file_prefix = 'part'\n",
-    "\n",
-    "def list_files_with_prefix(directory, prefix):\n",
-    "    file_paths = []\n",
-    "\n",
-    "    for root, _, files in os.walk(directory):\n",
-    "        for file in files:\n",
-    "            if file.startswith(prefix):\n",
-    "                absolute_path = os.path.join(root, file)\n",
-    "                file_paths.append(absolute_path)\n",
-    "\n",
-    "    return file_paths\n",
-    "\n",
-    "laion_file_paths = list_files_with_prefix(directory_path, file_prefix)\n",
-    "\n",
-    "dataframes = [dd.read_parquet(file) for file in laion_file_paths]\n",
-    "combined_dataframe = dd.multi.concat(dataframes)\n",
-    "\n",
-    "with ProgressBar():\n",
-    "    row_count = combined_dataframe.shape[0].compute()\n",
-    "    print(row_count)\n",
-    "\n",
-    "filtered_df = combined_dataframe[combined_dataframe['NSFW'] == \"UNLIKELY\"]\n",
-    "\n",
-    "num_samples = 225_000\n",
-    "selected_rows = filtered_df.sample(frac=num_samples / filtered_df.shape[0].compute())\n",
-    "\n",
-    "with ProgressBar():\n",
-    "    sampled_rows = selected_rows.compute()\n",
-    "\n",
-    "print(len(sampled_rows))\n",
-    "\n",
-    "with ProgressBar():\n",
-    "    selected_rows.to_parquet('laion_sampled', write_index=False)\n"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "bloom",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.16"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

src/images/Diffusion/scrape.py

@@ -1,149 +0,0 @@
-import argparse
-import time
-
-import polars as pl
-import requests
-
-
-def call_api(param):
-    url = "https://api.pullpush.io/reddit/search/submission/"
-    response = requests.get(url, params=param)
-    json_data = response.json()["data"]
-    create_utc = []
-    media_id = []
-    media_type_ls = []
-    post_ids = []
-    post_titles = []
-    cur_utc = 0
-    for submission in json_data:
-        cur_flair = submission["link_flair_text"]
-        cur_utc = submission["created_utc"]
-        media_ls = (
-            submission["media_metadata"]
-            if "media_metadata" in submission.keys()
-            else None
-        )
-        if param["flair"] is not None and cur_flair != param["flair"]:
-            continue
-        if media_ls is None:
-            continue
-        for id in media_ls.keys():
-            if media_ls[id]["status"] != "valid":
-                continue
-            try:
-                media_type = media_ls[id]["m"]
-            except:  # noqa
-                # video will error out
-                continue
-            if media_type == "image/png":
-                media_type_ls.append("png")
-            elif media_type == "image/jpg":
-                media_type_ls.append("jpg")
-            else:
-                continue
-            create_utc.append(int(cur_utc))
-            post_ids.append(submission["id"])
-            post_titles.append(submission["title"])
-            media_id.append(id)
-
-    df = pl.DataFrame(
-        {
-            "create_utc": create_utc,
-            "media_id": media_id,
-            "media_type": media_type_ls,
-            "post_id": post_ids,
-            "post_title": post_titles,
-        },
-        schema={
-            "create_utc": pl.Int64,
-            "media_id": pl.Utf8,
-            "media_type": pl.Utf8,
-            "post_id": pl.Utf8,
-            "post_title": pl.Utf8,
-        },
-    )
-    return df, int(cur_utc)
-
-
-def scraping_loop(
-    subreddit,
-    flair,
-    max_num=30000,
-    output_name=None,
-    before=None,
-):
-    collected_all = []
-    collected_len = 0
-    last_timestamp = int(time.time()) if before is None else before
-    param = {
-        "subreddit": subreddit,
-        "flair": flair,
-        "before": last_timestamp,
-    }
-    while collected_len < max_num:
-        collected_df, last_timestamp = call_api(param)
-        if collected_df.shape[0] == 0:
-            print("No more data, saving current data and exiting...")
-            break
-        collected_all.append(collected_df)
-        collected_len += collected_df.shape[0]
-        print(
-            f"collected_len: {collected_len}, "
-            f"last_timestamp: {last_timestamp}",
-        )
-        param["before"] = last_timestamp
-
-    df = pl.concat(collected_all)
-    df = (
-        df.with_columns(
-            pl.col("media_id")
-            .str.replace(r"^", "https://i.redd.it/")
-            .alias("url1"),
-            pl.col("create_utc")
-            .cast(pl.Int64)
-            .cast(pl.Utf8)
-            .str.to_datetime("%s")
-            .alias("time"),
-        )
-        .with_columns(
-            pl.col("media_type").str.replace(r"^", ".").alias("url2"),
-        )
-        .with_columns(
-            pl.concat_str(
-                [pl.col("url1"), pl.col("url2")],
-                separator="",
-            ).alias("url"),
-        )
-        .select("time", "url", "post_id", "post_title")
-    )
-    if output_name is None:
-        output_name = subreddit
-    df.write_parquet(f"urls/{output_name}.parquet")
-    df.select("url").write_csv(f"urls/{output_name}.csv", has_header=False)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--subreddit", help="subreddit name")
-    parser.add_argument("--flair", help="flair filter", default=None, type=str)
-    parser.add_argument(
-        "--max_num",
-        help="max number of posts to scrape",
-        default=30000,
-        type=int,
-    )
-    parser.add_argument(
-        "--output_name",
-        help="custom output name",
-        default=None,
-    )
-    parser.add_argument(
-        "--before",
-        help="before timestamp",
-        default=None,
-        type=int,
-    )
-
-    args = parser.parse_args()
-
-    scraping_loop(**args.__dict__)

src/images/Diffusion/utils_sampling.py

@@ -1,94 +0,0 @@
-import collections
-import random
-from typing import Callable
-
-from torchdata.datapipes.iter import IterDataPipe
-
-
-def get_second_entry(sample):
-    return sample[1]
-
-
-class UnderSamplerIterDataPipe(IterDataPipe):
-    """Dataset wrapper for under-sampling.
-
-    Copied from: https://github.com/MaxHalford/pytorch-resample/blob/master/pytorch_resample/under.py # noqa
-    Modified to work with multiple labels.
-
-    MIT License
-
-    Copyright (c) 2020 Max Halford
-
-    This method is based on rejection sampling.
-
-    Parameters:
-        dataset
-        desired_dist: The desired class distribution.
-            The keys are the classes whilst the
-            values are the desired class percentages.
-            The values are normalised so that sum up
-            to 1.
-        label_getter: A function that takes a sample and returns its label.
-        seed: Random seed for reproducibility.
-
-    Attributes:
-        actual_dist: The counts of the observed sample labels.
-        rng: A random number generator instance.
-
-    References:
-        - https://www.wikiwand.com/en/Rejection_sampling
-
-    """
-
-    def __init__(
-        self,
-        dataset: IterDataPipe,
-        desired_dist: dict,
-        label_getter: Callable = get_second_entry,
-        seed: int = None,
-    ):
-
-        self.dataset = dataset
-        self.desired_dist = {
-            c: p / sum(desired_dist.values()) for c, p in desired_dist.items()
-        }
-        self.label_getter = label_getter
-        self.seed = seed
-
-        self.actual_dist = collections.Counter()
-        self.rng = random.Random(seed)
-        self._pivot = None
-
-    def __iter__(self):
-
-        for dp in self.dataset:
-            y = self.label_getter(dp)
-
-            self.actual_dist[y] += 1
-
-            # To ease notation
-            f = self.desired_dist
-            g = self.actual_dist
-
-            # Check if the pivot needs to be changed
-            if y != self._pivot:
-                self._pivot = max(g.keys(), key=lambda y: f[y] / g[y])
-            else:
-                yield dp
-                continue
-
-            # Determine the sampling ratio if the observed label
-            # is not the pivot
-            M = f[self._pivot] / g[self._pivot]
-            ratio = f[y] / (M * g[y])
-
-            if ratio < 1 and self.rng.random() < ratio:
-                yield dp
-
-    @classmethod
-    def expected_size(cls, n, desired_dist, actual_dist):
-        M = max(
-            desired_dist.get(k) / actual_dist.get(k)
-            for k in set(desired_dist) | set(actual_dist)
-        )
-        return int(n / M)

src/images/Diffusion/visualizations.ipynb

@@ -1,196 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%pip install polars-lts-cpu"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "import polars as pl\n",
-    "import matplotlib.pyplot as plt\n",
-    "import seaborn as sns\n",
-    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def pfbeta(labels, predictions, beta=1):\n",
-    "    y_true_count = 0\n",
-    "    ctp = 0\n",
-    "    cfp = 0\n",
-    "\n",
-    "    for idx in range(len(labels)):\n",
-    "        prediction = min(max(predictions[idx], 0), 1)\n",
-    "        if (labels[idx]):\n",
-    "            y_true_count += 1\n",
-    "            ctp += prediction\n",
-    "        else:\n",
-    "            cfp += prediction\n",
-    "\n",
-    "    beta_squared = beta * beta\n",
-    "    c_precision = ctp / (ctp + cfp)\n",
-    "    c_recall = ctp / y_true_count\n",
-    "    if (c_precision > 0 and c_recall > 0):\n",
-    "        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n",
-    "        return result\n",
-    "    else:\n",
-    "        return 0\n",
-    "\n",
-    "def get_part_metrics(df: pl.DataFrame, threshold=0.3) -> dict:\n",
-    "    df = df.with_columns((df[\"preds\"] > threshold).alias(\"preds_bin\"))\n",
-    "    metrics = {}\n",
-    "    # binary metrics using the threshold\n",
-    "    metrics[\"accuracy\"] = accuracy_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    metrics[\"precision\"] = precision_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    metrics[\"recall\"] = recall_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    metrics[\"f1\"] = f1_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
-    "    # probabilistic F1 (doesn't depend on the threshold)\n",
-    "    metrics[\"pf1\"] = pfbeta(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
-    "    # ROC AUC\n",
-    "    metrics[\"roc_auc\"] = roc_auc_score(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
-    "    return metrics\n",
-    "\n",
-    "\n",
-    "def get_all_metrics(df: pl.DataFrame, threshold=0.3) -> pd.DataFrame:\n",
-    "    groups = [list(range(5)), [0, 1], [0, 4], [0, 2], [0, 3]]\n",
-    "    group_names = [\"all\", \"StableDiffusion\", \"Midjourney\", \"Dalle2\", \"Dalle3\"]\n",
-    "    all_metrics = []\n",
-    "    for i, g in enumerate(groups):\n",
-    "        subset = df.filter(pl.col(\"domains\").is_in(g))\n",
-    "        metrics = get_part_metrics(subset, threshold=threshold)\n",
-    "        metrics[\"group\"] = group_names[i]\n",
-    "        all_metrics.append(metrics)\n",
-    "    \n",
-    "    return pd.DataFrame(all_metrics)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Load the data from the output files\n",
-    "df1 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-1.csv')\n",
-    "df14 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-14.csv')\n",
-    "df142 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-142.csv')\n",
-    "df1423 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-1423.csv')\n",
-    "\n",
-    "metrics_df1 = get_all_metrics(df1, threshold=0.5)\n",
-    "metrics_df14 = get_all_metrics(df14, threshold=0.5)\n",
-    "metrics_df142 = get_all_metrics(df142, threshold=0.5)\n",
-    "metrics_df1423 = get_all_metrics(df1423, threshold=0.5)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metrics_df1.info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sns.set()\n",
-    "\n",
-    "models = ['StableDiffusion', 'Midjourney', 'Dalle2', 'Dalle3']\n",
-    "metrics = ['accuracy', 'f1', 'pf1', 'roc_auc']\n",
-    "\n",
-    "file_map = {\n",
-    "    ('StableDiffusion',): metrics_df1,\n",
-    "    ('StableDiffusion', 'Midjourney'): metrics_df14,\n",
-    "    ('StableDiffusion', 'Midjourney', 'Dalle2'): metrics_df142,\n",
-    "    ('StableDiffusion', 'Midjourney', 'Dalle2', 'Dalle3'): metrics_df1423,\n",
-    "}\n",
-    "\n",
-    "def create_heatmap_data(metric):\n",
-    "    data = pd.DataFrame(index=models[::-1], columns=models)\n",
-    "    for i, model_x in enumerate(models):\n",
-    "        for j, model_y in enumerate(models[::-1]):\n",
-    "            \n",
-    "            if i == 0:\n",
-    "                relevant_df = metrics_df1\n",
-    "            elif i == 1:\n",
-    "                relevant_df = metrics_df14\n",
-    "            elif i == 2:\n",
-    "                relevant_df = metrics_df142\n",
-    "            else:\n",
-    "                relevant_df = metrics_df1423\n",
-    "\n",
-    "            # Debugging: print the DataFrame being used and the model_y\n",
-    "            #print(f\"Using DataFrame for {models[:i+1]}, model_y: {model_y}\")\n",
-    "\n",
-    "            # Extract the metric value\n",
-    "            if model_y in relevant_df['group'].values:\n",
-    "                metric_value = relevant_df[relevant_df['group'] == model_y][metric].values[0]\n",
-    "                # Debugging: print the extracted metric value\n",
-    "                #print(f\"Metric value for {model_y}: {metric_value}\")\n",
-    "            else:\n",
-    "                metric_value = float('nan')  # Handle non-existent cases\n",
-    "                # Debugging: print a message for non-existent cases\n",
-    "                #print(f\"No data for combination: {model_x}, {model_y}\")\n",
-    "\n",
-    "            data.at[model_y, model_x] = metric_value\n",
-    "    \n",
-    "    for col in data.columns:\n",
-    "        data[col] = pd.to_numeric(data[col], errors='coerce')\n",
-    "\n",
-    "    # Debugging: print the final DataFrame\n",
-    "    # print(f\"Final Data for metric {metric}:\")\n",
-    "    # print(data)\n",
-    "    # print(data.dtypes)\n",
-    "    return data\n",
-    "\n",
-    "for metric in metrics:\n",
-    "    plt.figure(figsize=(10, 8))\n",
-    "    sns.heatmap(create_heatmap_data(metric), annot=True, cmap='coolwarm', fmt='.3f')\n",
-    "    plt.title(f\"Heatmap for {metric}\")\n",
-    "    plt.xlabel(\"Trained On (x-axis)\")\n",
-    "    plt.ylabel(\"Tested On (y-axis)\")\n",
-    "    plt.show()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "bloom",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.16"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

src/images/README.md

@@ -1,64 +0,0 @@
-# AI-generated image detection
-**(Work In Progress)**
-
-- [ ] Refactor code
-- [ ] Review dependencies
-- [ ] Containerize (Docker)
-- [ ] Update documentation
-
-## AI-Generated Image detection
-
-This part handles the detection of AI-generated images.
-The current code contains two classifiers to detect AI-generated images from two types of architectures:
-- GANs
-
-## Model weights
-
-### 1. CNN Detection
-
-Run the `download_weights_CNN.sh` script:
-
-```commandline
-bash download_weights_CNN.sh
-```
-
-Note: you need `wget` installed on your system (it is by default for most Linux systems).
-
-### 2. Diffusion
-
-**TODO**
-
-
-## Run the models
-
-Make sure you have the weights available before doing so.
-
-**TODO: environments**
-
-### 1. CNN Detection
-
-```commandline
-python CNN_model_classifier.py
-```
-Available options:
-
-- `-f / --file` (default=`'examples_realfakedir'`)
-- `-m / --model_path` (default=`'weights/blur_jpg_prob0.5.pth'`)
-- `-c / --crop` (default=`None`):  Specify crop size (int) by default, do not crop.
-- `--use_cpu`: use cpu (by default uses GPU) -> **TODO: remove (obsolete)**
-
-Example usage:
-
-```commandline
-python CNN_model_classifier.py -f examples/real.png -m weights/blur_jpg_prob0.5.pth
-```
-
-### 2. Diffusion detection
-
-**TODO**
-
-## References
-
-Based on:
-- https://github.com/hoangthuc701/GenAI-image-detection
-- https://github.com/ptmaimai106/DetectGenerateImageByRealImageOnly

src/images/Search_Image/Bing_search.py

@@ -1,93 +0,0 @@
-import json
-import os
-from dotenv import load_dotenv
-import requests
-
-# Load Bing Search API key
-load_dotenv() 
-BING_API_KEY = os.getenv("BING_API_KEY")
-
-def print_json(obj):
-    """Print the object as json"""
-    print(json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': ')))
-
-
-def get_image_urls(search_results):
-    """
-    Extracts image URLs from Bing Visual Search response.
-    Ref: https://learn.microsoft.com/en-us/bing/search-apis/bing-visual-search/how-to/search-response
-
-    Args:
-        search_results: A dict containing the Bing VisualSearch response data.
-
-    Returns:
-        A tuple containing two lists:
-            - List of image URLs from "PagesIncluding" section.
-            - List of image URLs from "VisualSearch" section (backup).
-    """
-    
-    pages_including_urls = []
-    visual_search_urls = []
-    
-    if "tags" not in search_results:
-        return pages_including_urls, visual_search_urls
-    
-    # Check for required keys directly
-    if not any(action.get("actions") for action in search_results["tags"]):
-        return pages_including_urls, visual_search_urls
-    
-    
-    for action in search_results["tags"]:
-        for result in action.get("actions", []):
-            # actions = PagesIncluding, main results
-            if result["name"] == "PagesIncluding":
-                pages_including_urls.extend(item["contentUrl"] for item in result["data"]["value"])
-            # actions = VisualSearch, back up results 
-            elif result["name"] == "VisualSearch":
-                visual_search_urls.extend(item["contentUrl"] for item in result["data"]["value"])
-
-    return pages_including_urls, visual_search_urls
-
-def reverse_image_search(image_path, subscription_key=BING_API_KEY):
-    """Performs a reverse image search using the Bing Visual Search API.
-
-    Args:
-        image_path: The path to the image file to search for.
-
-    Returns:
-        A list of image URLs found that are similar to the image in the 
-            specified path.
-
-    Raises:
-        requests.exceptions.RequestException: If the API request fails.
-    """
-    base_uri = "https://api.bing.microsoft.com/v7.0/images/visualsearch"
-    headers = {"Ocp-Apim-Subscription-Key": subscription_key}
-    
-    try:
-        files = {"image": ("image", open(image_path, "rb"))}
-        response = requests.post(base_uri, headers=headers, files=files)
-        response.raise_for_status()
-        search_results = response.json()
-        
-        return search_results
-
-    except requests.exceptions.RequestException as e:
-        raise requests.exceptions.RequestException(f"API request failed: {e}")
-    except OSError as e:
-        raise OSError(f"Error opening image file: {e}")
-
-if __name__ == "__main__":
-    # Example usage:
-    image_path = "data/test_data/human_news.jpg"
-    try:
-        search_results = reverse_image_search(image_path)
-        image_urls, backup_image_urls = get_image_urls(search_results)
-
-        # Print the results
-        print("Image URLs from PagesIncluding:")
-        print(image_urls)
-        print("\nImage URLs from VisualSearch (backup):")
-        print(backup_image_urls)
-    except Exception as e:
-        print(f"An error occurred: {e}")

src/images/Search_Image/image_model_share.py

@@ -1,142 +0,0 @@
-from sklearn.metrics import roc_auc_score
-from torchmetrics import Accuracy, Recall
-import pytorch_lightning as pl
-import timm  
-import torch
-from pytorch_lightning.callbacks import Model, EarlyStopping
-import logging
-from PIL import Image
-import torchvision.transforms as transforms
-from torchvision.transforms import v2
-
-logging.basicConfig(filename='training.log',filemode='w',level=logging.INFO, force=True)
-CHECKPOINT = "models/image_classifier/image-classifier-step=8008-val_loss=0.11.ckpt"
-
-
-
-class ImageClassifier(pl.LightningModule):
-    def __init__(self, lmd=0):
-        super().__init__()
-        self.model = timm.create_model('resnet50', pretrained=True, num_classes=1)
-        self.accuracy = Accuracy(task='binary', threshold=0.5)
-        self.recall = Recall(task='binary', threshold=0.5)  
-        self.validation_outputs = []
-        self.lmd = lmd
-
-    def forward(self, x):
-        return self.model(x)
-
-    def training_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-        
-        print(f"Shape of outputs (training): {outputs.shape}")
-        print(f"Shape of labels (training): {labels.shape}")
-        
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        logging.info(f"Training Step - ERM loss: {loss.item()}")
-        loss += self.lmd * (outputs ** 2).mean() # SD loss penalty
-        logging.info(f"Training Step - SD loss: {loss.item()}")
-        return loss
-
-    def validation_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        if outputs.shape == torch.Size([]):
-            return
-        
-        print(f"Shape of outputs (validation): {outputs.shape}")
-        print(f"Shape of labels (validation): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        preds = torch.sigmoid(outputs)
-        self.log('val_loss', loss, prog_bar=True, sync_dist=True)
-        self.log('val_acc', self.accuracy(preds, labels.int()), prog_bar=True, sync_dist=True)
-        self.log('val_recall', self.recall(preds, labels.int()), prog_bar=True, sync_dist=True)
-        output = {"val_loss": loss, "preds": preds, "labels": labels}
-        self.validation_outputs.append(output)
-        logging.info(f"Validation Step - Batch loss: {loss.item()}")
-        return output
-    
-    def predict_step(self, batch):
-        images, label, domain = batch
-        outputs = self.forward(images).squeeze()
-        preds = torch.sigmoid(outputs)
-        return preds, label, domain
-
-    def on_validation_epoch_end(self):
-        if not self.validation_outputs:
-            logging.warning("No outputs in validation step to process")
-            return
-        preds = torch.cat([x['preds'] for x in self.validation_outputs])
-        labels = torch.cat([x['labels'] for x in self.validation_outputs])
-        if labels.unique().size(0) == 1:
-            logging.warning("Only one class in validation step")
-            return
-        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
-        self.log('val_auc', auc_score, prog_bar=True, sync_dist=True)
-        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
-        self.validation_outputs = []
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
-        return optimizer
-
-
-
-def load_image(image_path, transform=None):
-    image = Image.open(image_path).convert('RGB')
-    
-    if transform:
-        image = transform(image)
-    
-    return image
-
-
-def predict_single_image(image_path, model, transform=None):
-    image = load_image(image_path, transform) 
-    
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    
-    model.to(device)
-    
-    image = image.to(device)
-
-    model.eval()
-    
-    with torch.no_grad():
-        image = image.unsqueeze(0)  
-        output = model(image).squeeze() 
-        print(output)
-        prediction = torch.sigmoid(output).item()  
-    
-    return prediction
-
-
-def image_generation_detection(image_path):
-    model = ImageClassifier.load_from_checkpoint(CHECKPOINT)
-
-    transform = v2.Compose([
-        transforms.ToTensor(),
-        v2.CenterCrop((256, 256)),
-    ])    
-
-    prediction = predict_single_image(image_path, model, transform) 
-    print("prediction",prediction)
-    
-    result = ""
-    if prediction <= 0.2:
-        result += "Most likely human"
-        image_prediction_label = "HUMAN"
-    else:
-        result += "Most likely machine"
-        image_prediction_label = "MACHINE"
-    image_confidence = min(1, 0.5 + abs(prediction - 0.2))
-    result += f" with confidence = {round(image_confidence * 100, 2)}%"
-    image_confidence = round(image_confidence * 100, 2)
-    return image_prediction_label, image_confidence
-
-
-if __name__ == "__main__":  
-    pass

src/images/Search_Image/search.py

@@ -1,56 +0,0 @@
-from google_img_source_search import ReverseImageSearcher
-import requests
-from io import BytesIO
-from PIL import Image
-import imagehash
-from google_img_source_search import ReverseImageSearcher
-
-def get_image_from_url(url):
-    response = requests.get(url)
-    return Image.open(BytesIO(response.content))
-
-def standardize_image(image):
-    # Convert to RGB if needed
-    if image.mode in ('RGBA', 'LA'):
-        background = Image.new('RGB', image.size, (255, 255, 255))
-        background.paste(image, mask=image.split()[-1])
-        image = background
-    elif image.mode != 'RGB':
-        image = image.convert('RGB')
-    
-    # Resize to standard size (e.g. 256x256)
-    standard_size = (256, 256)
-    image = image.resize(standard_size)
-    
-    return image
-
-def compare_images(image1, image2):
-    # Standardize both images first
-    img1_std = standardize_image(image1)
-    img2_std = standardize_image(image2)
-    
-    hash1 = imagehash.average_hash(img1_std)
-    hash2 = imagehash.average_hash(img2_std)
-    return hash1 - hash2  # Returns the Hamming distance between the hashes
-
-if __name__ == '__main__':
-    image_url = 'https://i.pinimg.com/originals/c4/50/35/c450352ac6ea8645ead206721673e8fb.png'
-    
-    # Get the image from URL
-    url_image = get_image_from_url(image_url)
-
-    # Search image
-    rev_img_searcher = ReverseImageSearcher()
-    res = rev_img_searcher.search(image_url)
-
-    for search_item in res:
-        print(f'Title: {search_item.page_title}')
-        # print(f'Site: {search_item.page_url}')
-        print(f'Img: {search_item.image_url}\n')
-
-        # Compare each search result image with the input image
-        result_image = get_image_from_url(search_item.image_url)
-        result_difference = compare_images(result_image, url_image)
-        print(f"Difference with search result: {result_difference}")
-        if result_difference == 0: 
-            break 

src/images/Search_Image/search_2.py

@@ -1,150 +0,0 @@
-import time
-import logging
-import requests
-from bs4 import BeautifulSoup
-from typing import Dict, Optional
-from urllib.parse import quote, urlparse
-
-logging.basicConfig(
-    filename='error.log',
-    level=logging.INFO,
-    format='%(asctime)s | [%(levelname)s]: %(message)s',
-    datefmt='%m-%d-%Y / %I:%M:%S %p'
-)
-
-class SearchResults:
-    def __init__(self, results):
-        self.results = results
-
-    def __str__(self):
-        output = ""
-        for result in self.results:
-            output += "---\n"
-            output += f"Title: {result.get('title', 'Title not found')}\n"
-            output += f"Link: {result.get('link', 'Link not found')}\n"
-            output += "---\n"
-        return output
-
-class GoogleReverseImageSearch:
-    def __init__(self):
-        self.base_url = "https://www.google.com/searchbyimage"
-        self.headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
-        self.retry_count = 3
-        self.retry_delay = 1
-
-    def response(self, query: str, image_url: str, max_results: int = 10, delay: int = 1) -> SearchResults:
-        self._validate_input(query, image_url)
-        
-        encoded_query = quote(query)
-        encoded_image_url = quote(image_url)
-
-        url = f"{self.base_url}?q={encoded_query}&image_url={encoded_image_url}&sbisrc=cr_1_5_2"
-
-        all_results = []
-        start_index = 0
-
-        while len(all_results) < max_results:
-            if start_index != 0:
-                time.sleep(delay)
-                
-            paginated_url = f"{url}&start={start_index}"
-
-            response = self._make_request(paginated_url)
-            if response is None:
-                break
-
-            search_results, valid_content = self._parse_search_results(response.text)
-            if not valid_content:
-                logging.warning("Unexpected HTML structure encountered.")
-                break
-
-            for result in search_results:
-                if len(all_results) >= max_results:
-                    break
-                data = self._extract_result_data(result)
-                if data and data not in all_results:
-                    all_results.append(data)
-
-            start_index += (len(all_results)-start_index)
-
-        if len(all_results) == 0:
-            logging.warning(f"No results were found for the given query: [{query}], and/or image URL: [{image_url}].")
-            return "No results found. Please try again with a different query and/or image URL."
-        else:
-            return SearchResults(all_results[:max_results])
-    
-    def _validate_input(self, query: str, image_url: str):
-        if not query:
-            raise ValueError("Query not found. Please enter a query and try again.")
-        if not image_url:
-            raise ValueError("Image URL not found. Please enter an image URL and try again.")
-        if not self._validate_image_url(image_url):
-            raise ValueError("Invalid image URL. Please enter a valid image URL and try again.")
-    
-    def _validate_image_url(self, url: str) -> bool:
-        parsed_url = urlparse(url)
-        path = parsed_url.path.lower()
-        valid_extensions = (".jpg", ".jpeg", ".png", ".webp")
-        return any(path.endswith(ext) for ext in valid_extensions)
-    
-    def _make_request(self, url: str):
-        attempts = 0
-        while attempts < self.retry_count:
-            try:
-                response = requests.get(url, headers=self.headers)
-                if response.headers.get('Content-Type', '').startswith('text/html'):
-                    response.raise_for_status()
-                    return response
-                else:
-                    logging.warning("Non-HTML content received.")
-                    return None
-            except requests.exceptions.HTTPError as http_err:
-                logging.error(f"HTTP error occurred: {http_err}")
-                attempts += 1
-                time.sleep(self.retry_delay)
-            except Exception as err:
-                logging.error(f"An error occurred: {err}")
-                return None
-        return None
-
-    def _parse_search_results(self, html_content: str) -> (Optional[list], bool):
-        try:
-            soup = BeautifulSoup(html_content, "html.parser")
-            return soup.find_all('div', class_='g'), True
-        except Exception as e:
-            logging.error(f"Error parsing HTML content: {e}")
-            return None, False
-
-    def _extract_result_data(self, result) -> Dict:
-        link = result.find('a', href=True)['href'] if result.find('a', href=True) else None
-        title = result.find('h3').get_text(strip=True) if result.find('h3') else None
-        return {"link": link, "title": title} if link and title else {}
-    
-
-if __name__ == "__main__":
-    # request = GoogleReverseImageSearch()
-
-    # response = request.response(
-    #     query="Example Query",
-    #     image_url="https://ichef.bbci.co.uk/images/ic/1024xn/p0khzhhl.jpg.webp",
-    #     max_results=5
-    # )
-
-    # print(response)
-
-    # Path to local image
-    image_path = "data/test_data/towel.jpg"
-    image_path = "C:\\TTProjects\\prj-nict-ai-content-detection\\data\\test_data\\towel.jpg"
-
-    import json
-    file_path = image_path
-    search_url = 'https://yandex.ru/images/search'
-    files = {'upfile': ('blob', open(file_path, 'rb'), 'image/jpeg')}
-    params = {'rpt': 'imageview', 'format': 'json', 'request': '{"blocks":[{"block":"b-page_type_search-by-image__link"}]}'}
-    response = requests.post(search_url, params=params, files=files)
-    query_string = json.loads(response.content)['blocks'][0]['params']['url']
-    img_search_url = search_url + '?' + query_string
-    print(img_search_url)
-
-    response = requests.get(img_search_url)
-    print(response.text)

src/images/Search_Image/search_yandex.py

@@ -1,177 +0,0 @@
-import time
-import logging
-import requests
-from bs4 import BeautifulSoup
-from typing import Dict, Optional
-from urllib.parse import quote, urlparse
-
-logging.basicConfig(
-    filename='error.log',
-    level=logging.INFO,
-    format='%(asctime)s | [%(levelname)s]: %(message)s',
-    datefmt='%m-%d-%Y / %I:%M:%S %p'
-)
-
-class SearchResults:
-    def __init__(self, results):
-        self.results = results
-
-    def __str__(self):
-        output = ""
-        for result in self.results:
-            output += "---\n"
-            output += f"Title: {result.get('title', 'Title not found')}\n"
-            output += f"Link: {result.get('link', 'Link not found')}\n"
-            output += "---\n"
-        return output
-
-class ReverseImageSearch:
-    def __init__(self):
-        self.base_url = "https://yandex.ru/images/search"
-        self.headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
-        self.retry_count = 3
-        self.retry_delay = 1
-
-    def response(self, query: str, image_url: str, max_results: int = 10, delay: int = 1) -> SearchResults:
-        self._validate_input(query, image_url)
-        
-        encoded_query = quote(query)
-        encoded_image_url = quote(image_url)
-
-        url = f"{self.base_url}?q={encoded_query}&image_url={encoded_image_url}&sbisrc=cr_1_5_2"
-
-        all_results = []
-        start_index = 0
-
-        while len(all_results) < max_results:
-            if start_index != 0:
-                time.sleep(delay)
-                
-            paginated_url = f"{url}&start={start_index}"
-
-            response = self._make_request(paginated_url)
-            if response is None:
-                break
-
-            search_results, valid_content = self._parse_search_results(response.text)
-            if not valid_content:
-                logging.warning("Unexpected HTML structure encountered.")
-                break
-
-            for result in search_results:
-                if len(all_results) >= max_results:
-                    break
-                data = self._extract_result_data(result)
-                if data and data not in all_results:
-                    all_results.append(data)
-
-            start_index += (len(all_results)-start_index)
-
-        if len(all_results) == 0:
-            logging.warning(f"No results were found for the given query: [{query}], and/or image URL: [{image_url}].")
-            return "No results found. Please try again with a different query and/or image URL."
-        else:
-            return SearchResults(all_results[:max_results])
-    
-    def _validate_input(self, query: str, image_url: str):
-        if not query:
-            raise ValueError("Query not found. Please enter a query and try again.")
-        if not image_url:
-            raise ValueError("Image URL not found. Please enter an image URL and try again.")
-        if not self._validate_image_url(image_url):
-            raise ValueError("Invalid image URL. Please enter a valid image URL and try again.")
-    
-    def _validate_image_url(self, url: str) -> bool:
-        parsed_url = urlparse(url)
-        path = parsed_url.path.lower()
-        valid_extensions = (".jpg", ".jpeg", ".png", ".webp")
-        return any(path.endswith(ext) for ext in valid_extensions)
-    
-    def _make_request(self, url: str):
-        attempts = 0
-        while attempts < self.retry_count:
-            try:
-                response = requests.get(url, headers=self.headers)
-                if response.headers.get('Content-Type', '').startswith('text/html'):
-                    response.raise_for_status()
-                    return response
-                else:
-                    logging.warning("Non-HTML content received.")
-                    return None
-            except requests.exceptions.HTTPError as http_err:
-                logging.error(f"HTTP error occurred: {http_err}")
-                attempts += 1
-                time.sleep(self.retry_delay)
-            except Exception as err:
-                logging.error(f"An error occurred: {err}")
-                return None
-        return None
-
-    def _parse_search_results(self, html_content: str) -> (Optional[list], bool):
-        try:
-            soup = BeautifulSoup(html_content, "html.parser")
-            return soup.find_all('div', class_='g'), True
-        except Exception as e:
-            logging.error(f"Error parsing HTML content: {e}")
-            return None, False
-
-    def _extract_result_data(self, result) -> Dict:
-        link = result.find('a', href=True)['href'] if result.find('a', href=True) else None
-        title = result.find('h3').get_text(strip=True) if result.find('h3') else None
-        return {"link": link, "title": title} if link and title else {}
-    
-def yandex_reverse_image_search(image_url):
-    # Simulate a user agent to avoid being blocked
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
-
-    try:
-        response = requests.get(image_url, headers=headers)
-        response.raise_for_status()  # Raise an exception for bad status codes
-
-        # Parse the HTML content
-        soup = BeautifulSoup(response.content, 'html.parser')
-
-        # Extract image URLs (example - adapt based on Yandex's HTML structure)
-        image_urls = [img['src'] for img in soup.find_all('img')]
-
-        # Extract related searches (example - adapt based on Yandex's HTML structure)
-        related_searches = [text for text in soup.find_all(class_="related-searches")] 
-
-        return image_urls, related_searches
-
-    except requests.exceptions.RequestException as e:
-        print(f"Error fetching image: {e}")
-        return [], []
-
-
-if __name__ == "__main__":
-    # request = GoogleReverseImageSearch()
-
-    # response = request.response(
-    #     query="Example Query",
-    #     image_url="https://ichef.bbci.co.uk/images/ic/1024xn/p0khzhhl.jpg.webp",
-    #     max_results=5
-    # )
-
-    # print(response)
-
-    # Path to local image
-    image_path = "data/test_data/towel.jpg"
-    image_path = "C:\\TTProjects\\prj-nict-ai-content-detection\\data\\test_data\\towel.jpg"
-
-    import json
-    file_path = image_path
-    search_url = 'https://yandex.ru/images/search'
-    files = {'upfile': ('blob', open(file_path, 'rb'), 'image/jpeg')}
-    params = {'rpt': 'imageview', 'format': 'json', 'request': '{"blocks":[{"block":"b-page_type_search-by-image__link"}]}'}
-    response = requests.post(search_url, params=params, files=files)
-    query_string = json.loads(response.content)['blocks'][0]['params']['url']
-    img_search_url = search_url + '?' + query_string
-    print(img_search_url)
-
-    image_urls, related_searches = yandex_reverse_image_search(img_search_url)
-
-    print("Image URLs:", image_urls)
-    print("Related Searches:", related_searches)
-
-    

src/images/diffusion_data_loader.py

@@ -1,229 +0,0 @@
-import argparse
-import collections
-import random
-from typing import Iterator
-
-import cv2
-import numpy as np
-import torchdata.datapipes as dp
-from imwatermark import WatermarkEncoder
-from PIL import (
-    Image,
-    ImageFile,
-)
-from torch.utils.data import DataLoader
-from torchdata.datapipes.iter import (
-    Concater,
-    FileLister,
-    FileOpener,
-    SampleMultiplexer,
-)
-from torchvision.transforms import v2
-from tqdm import tqdm
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-Image.MAX_IMAGE_PIXELS = 1000000000
-
-encoder = WatermarkEncoder()
-encoder.set_watermark("bytes", b"test")
-
-DOMAIN_LABELS = {
-    0: "laion",
-    1: "StableDiffusion",
-    2: "dalle2",
-    3: "dalle3",
-    4: "midjourney",
-}
-
-N_SAMPLES = {
-    0: (115346, 14418, 14419),
-    1: (22060, 2757, 2758),
-    4: (21096, 2637, 2637),
-    2: (13582, 1697, 1699),
-    3: (12027, 1503, 1504),
-}
-
-
-@dp.functional_datapipe("collect_from_workers")
-class WorkerResultCollector(dp.iter.IterDataPipe):
-    def __init__(self, source: dp.iter.IterDataPipe):
-        self.source = source
-
-    def __iter__(self) -> Iterator:
-        yield from self.source
-
-    def is_replicable(self) -> bool:
-        """Method to force data back to main process"""
-        return False
-
-
-def crop_bottom(image, cutoff=16):
-    return image[:, :-cutoff, :]
-
-
-def random_gaussian_blur(image, p=0.01):
-    if random.random() < p:
-        return v2.functional.gaussian_blur(image, kernel_size=5)
-    return image
-
-
-def random_invisible_watermark(image, p=0.2):
-    image_np = np.array(image)
-    image_np = np.transpose(image_np, (1, 2, 0))
-
-    if image_np.ndim == 2:  # Grayscale image
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)
-    elif image_np.shape[2] == 4:  # RGBA image
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR)
-
-    if image_np.shape[0] < 256 or image_np.shape[1] < 256:
-        image_np = cv2.resize(
-            image_np,
-            (256, 256),
-            interpolation=cv2.INTER_AREA,
-        )
-
-    if random.random() < p:
-        return encoder.encode(image_np, method="dwtDct")
-
-    return image_np
-
-
-def build_transform(split: str):
-    train_transform = v2.Compose(
-        [
-            v2.Lambda(crop_bottom),
-            v2.RandomCrop((256, 256), pad_if_needed=True),
-            v2.Lambda(random_gaussian_blur),
-            v2.RandomGrayscale(p=0.05),
-            v2.Lambda(random_invisible_watermark),
-            v2.ToImage(),
-        ],
-    )
-
-    eval_transform = v2.Compose(
-        [
-            v2.CenterCrop((256, 256)),
-        ],
-    )
-    transform = train_transform if split == "train" else eval_transform
-
-    return transform
-
-
-def dp_to_tuple_train(input_dict):
-    transform = build_transform("train")
-    return (
-        transform(input_dict[".jpg"]),
-        input_dict[".label.cls"],
-        input_dict[".domain_label.cls"],
-    )
-
-
-def dp_to_tuple_eval(input_dict):
-    transform = build_transform("eval")
-    return (
-        transform(input_dict[".jpg"]),
-        input_dict[".label.cls"],
-        input_dict[".domain_label.cls"],
-    )
-
-
-def load_dataset(domains: list[int], split: str):
-    laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar")
-    genai_lister = {
-        d: FileLister(
-            f"./data/genai-images/{DOMAIN_LABELS[d]}",
-            f"{split}*.tar",
-        )
-        for d in domains
-        if DOMAIN_LABELS[d] != "laion"
-    }
-    weight_genai = 1 / len(genai_lister)
-
-    def open_lister(lister):
-        opener = FileOpener(lister, mode="b")
-        return opener.load_from_tar().routed_decode().webdataset()
-
-    buffer_size1 = 100 if split == "train" else 10
-    buffer_size2 = 100 if split == "train" else 10
-
-    if split != "train":
-        all_lister = [laion_lister] + list(genai_lister.values())
-        dp = open_lister(Concater(*all_lister)).sharding_filter()
-    else:
-        laion_dp = (
-            open_lister(laion_lister.shuffle())
-            .cycle()
-            .sharding_filter()
-            .shuffle(buffer_size=buffer_size1)
-        )
-        genai_dp = {
-            open_lister(genai_lister[d].shuffle())
-            .cycle()
-            .sharding_filter()
-            .shuffle(
-                buffer_size=buffer_size1,
-            ): weight_genai
-            for d in domains
-            if DOMAIN_LABELS[d] != "laion"
-        }
-        dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle(
-            buffer_size=buffer_size2,
-        )
-
-    if split == "train":
-        dp = dp.map(dp_to_tuple_train)
-    else:
-        dp = dp.map(dp_to_tuple_eval)
-
-    return dp
-
-
-def load_dataloader(
-    domains: list[int],
-    split: str,
-    batch_size: int = 32,
-    num_workers: int = 4,
-):
-    dp = load_dataset(domains, split)
-    # if split == "train":
-    #     dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42)
-    dp = dp.batch(batch_size).collate()
-    dl = DataLoader(
-        dp,
-        batch_size=None,
-        num_workers=num_workers,
-        pin_memory=True,
-    )
-
-    return dl
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    args = parser.parse_args()
-
-    # testing code
-    dl = load_dataloader([0, 1], "train", num_workers=8)
-    y_dist = collections.Counter()
-    d_dist = collections.Counter()
-
-    for i, (img, y, d) in tqdm(enumerate(dl)):
-        if i % 100 == 0:
-            print(y, d)
-        if i == 400:
-            break
-        y_dist.update(y.numpy())
-        d_dist.update(d.numpy())
-
-    print("class label")
-    for label in sorted(y_dist):
-        frequency = y_dist[label] / sum(y_dist.values())
-        print(f"• {label}: {frequency:.2%} ({y_dist[label]})")
-
-    print("domain label")
-    for label in sorted(d_dist):
-        frequency = d_dist[label] / sum(d_dist.values())
-        print(f"• {label}: {frequency:.2%} ({d_dist[label]})")

src/images/diffusion_model_classifier.py

@@ -1,293 +0,0 @@
-import argparse
-import logging
-import os
-
-import pandas as pd
-import pytorch_lightning as pl
-import timm
-import torch
-import torch.nn.functional as F
-import torchvision.transforms as transforms
-from PIL import Image
-from pytorch_lightning.callbacks import (
-    EarlyStopping,
-    ModelCheckpoint,
-)
-from sklearn.metrics import roc_auc_score
-from torchmetrics import (
-    Accuracy,
-    Recall,
-)
-
-from .diffusion_data_loader import load_dataloader
-
-
-class ImageClassifier(pl.LightningModule):
-    def __init__(self, lmd=0):
-        super().__init__()
-        self.model = timm.create_model(
-            "resnet50",
-            pretrained=True,
-            num_classes=1,
-        )
-        self.accuracy = Accuracy(task="binary", threshold=0.5)
-        self.recall = Recall(task="binary", threshold=0.5)
-        self.validation_outputs = []
-        self.lmd = lmd
-
-    def forward(self, x):
-        return self.model(x)
-
-    def training_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        print(f"Shape of outputs (training): {outputs.shape}")
-        print(f"Shape of labels (training): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        logging.info(f"Training Step - ERM loss: {loss.item()}")
-        loss += self.lmd * (outputs**2).mean()  # SD loss penalty
-        logging.info(f"Training Step - SD loss: {loss.item()}")
-        return loss
-
-    def validation_step(self, batch):
-        images, labels, _ = batch
-        outputs = self.forward(images).squeeze()
-
-        if outputs.shape == torch.Size([]):
-            return
-
-        print(f"Shape of outputs (validation): {outputs.shape}")
-        print(f"Shape of labels (validation): {labels.shape}")
-
-        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
-        preds = torch.sigmoid(outputs)
-        self.log("val_loss", loss, prog_bar=True, sync_dist=True)
-        self.log(
-            "val_acc",
-            self.accuracy(preds, labels.int()),
-            prog_bar=True,
-            sync_dist=True,
-        )
-        self.log(
-            "val_recall",
-            self.recall(preds, labels.int()),
-            prog_bar=True,
-            sync_dist=True,
-        )
-        output = {"val_loss": loss, "preds": preds, "labels": labels}
-        self.validation_outputs.append(output)
-        logging.info(f"Validation Step - Batch loss: {loss.item()}")
-        return output
-
-    def predict_step(self, batch):
-        images, label, domain = batch
-        outputs = self.forward(images).squeeze()
-        preds = torch.sigmoid(outputs)
-        return preds, label, domain
-
-    def on_validation_epoch_end(self):
-        if not self.validation_outputs:
-            logging.warning("No outputs in validation step to process")
-            return
-        preds = torch.cat([x["preds"] for x in self.validation_outputs])
-        labels = torch.cat([x["labels"] for x in self.validation_outputs])
-        if labels.unique().size(0) == 1:
-            logging.warning("Only one class in validation step")
-            return
-        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
-        self.log("val_auc", auc_score, prog_bar=True, sync_dist=True)
-        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
-        self.validation_outputs = []
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
-        return optimizer
-
-
-def load_image(image_path, transform=None):
-    image = Image.open(image_path).convert("RGB")
-
-    if transform:
-        image = transform(image)
-
-    return image
-
-
-def predict_single_image(image, model):
-
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-    model.to(device)
-
-    image = image.to(device)
-
-    model.eval()
-
-    with torch.no_grad():
-        image = image.unsqueeze(0)
-        output = model(image).squeeze()
-        prediction = torch.sigmoid(output).item()
-
-    return prediction
-
-
-if __name__ == "__main__":
-    checkpoint_callback = ModelCheckpoint(
-        monitor="val_loss",
-        dirpath="./model_checkpoints/",
-        filename="image-classifier-{step}-{val_loss:.2f}",
-        save_top_k=3,
-        mode="min",
-        every_n_train_steps=1001,
-        enable_version_counter=True,
-    )
-
-    early_stop_callback = EarlyStopping(
-        monitor="val_loss",
-        patience=4,
-        mode="min",
-    )
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--ckpt_path",
-        help="checkpoint to continue from",
-        required=False,
-    )
-    parser.add_argument(
-        "--predict",
-        help="predict on test set",
-        action="store_true",
-    )
-    parser.add_argument("--reset", help="reset training", action="store_true")
-    parser.add_argument(
-        "--predict_image",
-        help="predict the class of a single image",
-        action="store_true",
-    )
-    parser.add_argument(
-        "--image_path",
-        help="path to the image to predict",
-        type=str,
-        required=False,
-    )
-    parser.add_argument(
-        "--dir",
-        help="path to the images to predict",
-        type=str,
-        required=False,
-    )
-    parser.add_argument(
-        "--output_file",
-        help="path to output file",
-        type=str,
-        required=False,
-    )
-    args = parser.parse_args()
-
-    train_domains = [0, 1, 4]
-    val_domains = [0, 1, 4]
-    lmd_value = 0
-
-    if args.predict:
-        test_dl = load_dataloader(
-            [0, 1, 2, 3, 4],
-            "test",
-            batch_size=10,
-            num_workers=1,
-        )
-        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-        trainer = pl.Trainer()
-        predictions = trainer.predict(model, dataloaders=test_dl)
-        preds, labels, domains = zip(*predictions)
-        preds = torch.cat(preds).cpu().numpy()
-        labels = torch.cat(labels).cpu().numpy()
-        domains = torch.cat(domains).cpu().numpy()
-        print(preds.shape, labels.shape, domains.shape)
-        df = pd.DataFrame(
-            {"preds": preds, "labels": labels, "domains": domains},
-        )
-        filename = "preds-" + args.ckpt_path.split("/")[-1]
-        df.to_csv(f"outputs/{filename}.csv", index=False)
-    elif args.predict_image:
-        image_path = args.image_path
-        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-
-        # Define the transformations for the image
-        transform = transforms.Compose(
-            [
-                transforms.CenterCrop((256, 256)),
-                transforms.ToTensor(),
-            ],
-        )
-        image = load_image(image_path, transform)
-        prediction = predict_single_image(image, model)
-        print("prediction", prediction)
-
-        # Output the prediction
-        print(
-            f"Prediction for {image_path}: "
-            f"{'Human' if prediction <= 0.05 else 'Generated'}",
-        )
-    elif args.dir is not None:
-        predictions = []
-        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-        transform = transforms.Compose(
-            [
-                transforms.CenterCrop((256, 256)),
-                transforms.ToTensor(),
-            ],
-        )
-        for root, dirs, files in os.walk(os.path.abspath(args.dir)):
-            for f_name in files:
-                f = os.path.join(root, f_name)
-                print(f"Predicting: {f}")
-                p = predict_single_image(f, model)
-                predictions.append([f, f.split("/")[-2], p, p > 0.5])
-                print(f"--predicted: {p}")
-
-        df = pd.DataFrame(
-            predictions,
-            columns=["path", "folder", "pred", "class"],
-        )
-        df.to_csv(args.output_file, index=False)
-    else:
-        logging.basicConfig(
-            filename="training.log",
-            filemode="w",
-            level=logging.INFO,
-            force=True,
-        )
-        train_dl = load_dataloader(
-            train_domains,
-            "train",
-            batch_size=128,
-            num_workers=4,
-        )
-        logging.info("Training dataloader loaded")
-        val_dl = load_dataloader(
-            val_domains,
-            "val",
-            batch_size=128,
-            num_workers=4,
-        )
-        logging.info("Validation dataloader loaded")
-
-        if args.reset:
-            model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
-        else:
-            model = ImageClassifier(lmd=lmd_value)
-        trainer = pl.Trainer(
-            callbacks=[checkpoint_callback, early_stop_callback],
-            max_steps=20000,
-            val_check_interval=1000,
-            check_val_every_n_epoch=None,
-        )
-        trainer.fit(
-            model=model,
-            train_dataloaders=train_dl,
-            val_dataloaders=val_dl,
-            ckpt_path=args.ckpt_path if not args.reset else None,
-        )

src/images/diffusion_utils_sampling.py

@@ -1,94 +0,0 @@
-import collections
-import random
-from typing import Callable
-
-from torchdata.datapipes.iter import IterDataPipe
-
-
-def get_second_entry(sample):
-    return sample[1]
-
-
-class UnderSamplerIterDataPipe(IterDataPipe):
-    """Dataset wrapper for under-sampling.
-
-    Copied from: https://github.com/MaxHalford/pytorch-resample/blob/master/pytorch_resample/under.py # noqa
-    Modified to work with multiple labels.
-
-    MIT License
-
-    Copyright (c) 2020 Max Halford
-
-    This method is based on rejection sampling.
-
-    Parameters:
-        dataset
-        desired_dist: The desired class distribution.
-            The keys are the classes whilst the
-            values are the desired class percentages.
-            The values are normalised so that sum up
-            to 1.
-        label_getter: A function that takes a sample and returns its label.
-        seed: Random seed for reproducibility.
-
-    Attributes:
-        actual_dist: The counts of the observed sample labels.
-        rng: A random number generator instance.
-
-    References:
-        - https://www.wikiwand.com/en/Rejection_sampling
-
-    """
-
-    def __init__(
-        self,
-        dataset: IterDataPipe,
-        desired_dist: dict,
-        label_getter: Callable = get_second_entry,
-        seed: int = None,
-    ):
-
-        self.dataset = dataset
-        self.desired_dist = {
-            c: p / sum(desired_dist.values()) for c, p in desired_dist.items()
-        }
-        self.label_getter = label_getter
-        self.seed = seed
-
-        self.actual_dist = collections.Counter()
-        self.rng = random.Random(seed)
-        self._pivot = None
-
-    def __iter__(self):
-
-        for dp in self.dataset:
-            y = self.label_getter(dp)
-
-            self.actual_dist[y] += 1
-
-            # To ease notation
-            f = self.desired_dist
-            g = self.actual_dist
-
-            # Check if the pivot needs to be changed
-            if y != self._pivot:
-                self._pivot = max(g.keys(), key=lambda y: f[y] / g[y])
-            else:
-                yield dp
-                continue
-
-            # Determine the sampling ratio if the observed label
-            # is not the pivot
-            M = f[self._pivot] / g[self._pivot]
-            ratio = f[y] / (M * g[y])
-
-            if ratio < 1 and self.rng.random() < ratio:
-                yield dp
-
-    @classmethod
-    def expected_size(cls, n, desired_dist, actual_dist):
-        M = max(
-            desired_dist.get(k) / actual_dist.get(k)
-            for k in set(desired_dist) | set(actual_dist)
-        )
-        return int(n / M)

src/images/image_demo.py

@@ -1,73 +0,0 @@
-import gradio as gr
-import torchvision.transforms as transforms
-from CNN_model_classifier import predict_cnn
-from diffusion_model_classifier import (
-    ImageClassifier,
-    predict_single_image,
-)
-
-gr.set_static_paths(paths=["samples/"])
-diffusion_model = (
-    "Diffusion/model_checkpoints/image-classifier-step=7007-val_loss=0.09.ckpt"
-)
-cnn_model = "CNN/model_checkpoints/blur_jpg_prob0.5.pth"
-
-
-def get_prediction_diffusion(image):
-    model = ImageClassifier.load_from_checkpoint(diffusion_model)
-
-    prediction = predict_single_image(image, model)
-    print(prediction)
-    return (prediction >= 0.001, prediction)
-
-
-def get_prediction_cnn(image):
-    prediction = predict_cnn(image, cnn_model)
-    return (prediction >= 0.5, prediction)
-
-
-def predict(inp):
-    # Define the transformations for the image
-    transform = transforms.Compose(
-        [
-            transforms.Resize((224, 224)),  # Image size expected by ResNet50
-            transforms.ToTensor(),
-            transforms.Normalize(
-                mean=[0.485, 0.456, 0.406],
-                std=[0.229, 0.224, 0.225],
-            ),
-        ],
-    )
-    image_tensor = transform(inp)
-    pred_diff, prob_diff = get_prediction_diffusion(image_tensor)
-    pred_cnn, prob_cnn = get_prediction_cnn(image_tensor)
-    verdict = (
-        "AI Generated" if (pred_diff or pred_cnn) else "No GenAI detected"
-    )
-    return (
-        f"<h1>{verdict}</h1>"
-        f"<ul>"
-        f"<li>Diffusion detection score: {prob_diff:.2} "
-        f"{'(MATCH)' if pred_diff else ''}</li>"
-        f"<li>CNN detection score: {prob_cnn:.1%} "
-        f"{'(MATCH)' if pred_cnn else ''}</li>"
-        f"</ul>"
-    )
-
-
-demo = gr.Interface(
-    title="AI-generated image detection",
-    description="Demo by NICT & Tokyo Techies ",
-    fn=predict,
-    inputs=gr.Image(type="pil"),
-    outputs=gr.HTML(),
-    examples=[
-        ["samples/fake_dalle.jpg", "Generated (Dall-E)"],
-        ["samples/fake_midjourney.png", "Generated (MidJourney)"],
-        ["samples/fake_stable.jpg", "Generated (Stable Diffusion)"],
-        ["samples/fake_cnn.png", "Generated (GAN)"],
-        ["samples/real.png", "Organic"],
-    ],
-)
-
-demo.launch()

src/main.py

@@ -1,51 +0,0 @@
-from texts.models import TextDetector
-
-
-def extract_text_and_images(path: str):
-    text_content = ""
-    image_paths = ""
-    return text_content, image_paths
-
-
-def process_document(document_path) -> list:
-    """
-    Processes a given document, separating text and images,
-    and then analyzes them.
-
-    Args:
-        document_path: Path to the document.
-
-    Returns:
-        A list containing the AI content percentage for text and images.
-    """
-
-    # Extract text and images from the document
-    text_content, image_paths = extract_text_and_images(document_path)
-
-    # Analyze text content
-    text_detector = TextDetector()
-    text_ai_content_percentage = text_detector.analyze_text(text_content)
-
-    # Analyze image content
-    image_ai_content_percentages = []
-    for image_path in image_paths:
-        # TODO: add image_detector class
-        # image_ai_content = image_detector.analyze_image(image_path)
-        image_ai_content = 100
-        image_ai_content_percentages.append(image_ai_content)
-
-    return [text_ai_content_percentage, image_ai_content_percentages]
-
-
-def main():
-    document_path = "../data.pdf"  # Replace with your document path
-    text_ai_content_percentage, image_ai_content_percentages = (
-        process_document(document_path)
-    )
-
-    print("Text AI Content Percentage:", text_ai_content_percentage)
-    print("Combined AI Content Percentage:", image_ai_content_percentages)
-
-
-if __name__ == "__main__":
-    main()

src/texts/MAGE/.gradio/flagged/dataset1.csv

@@ -1,2 +0,0 @@
-input text,AI-text detection,timestamp
-Does Chicago have any stores and does Joe live here?,"[{""token"": ""Does Chicago have any stores and does Joe live here?"", ""class_or_confidence"": ""human-written""}]",2024-12-09 13:40:10.255451

src/texts/MAGE/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

src/texts/MAGE/README.md

@@ -1,258 +0,0 @@
-<div align="center">
-<p align="center">
-  <img src="./figures/intro.png" width="75%" height="75%" />
-</p>
-</div>
-
-<div align="center">
-<h1><img src="./figures/title.png" width="30px" height="30px" style="display:inline;margin-right:10px;">MAGE: Machine-generated Text Detection in the Wild</h1>
-</div>
-
-<div align="center">
-<img src="https://img.shields.io/badge/Version-1.0.0-blue.svg" alt="Version"> 
-<img src="https://img.shields.io/badge/License-CC%20BY%204.0-green.svg" alt="License">
-<img src="https://img.shields.io/github/stars/yafuly/DeepfakeTextDetect?color=yellow" alt="Stars">
-<img src="https://img.shields.io/github/issues/yafuly/DeepfakeTextDetect?color=red" alt="Issues">
-
-<!-- **Authors:** -->
-<br>
-
-_**Yafu Li<sup>†</sup><sup>‡</sup>, Qintong Li<sup>§</sup>, Leyang Cui<sup>¶</sup>, Wei Bi<sup>¶</sup>,Zhilin Wang<sup>$</sup><br>**_
-
-_**Longyue Wang<sup>¶</sup>, Linyi Yang<sup>‡</sup>, Shuming Shi<sup>¶</sup>, Yue Zhang<sup>‡</sup><br>**_
-
-<!-- **Affiliations:** -->
-
-_<sup>†</sup> Zhejiang University,
-<sup>‡</sup> Westlake University,
-<sup>§</sup> The University of Hong Kong,
-<sup>$</sup> Jilin University,
-<sup>¶</sup> Tencent AI Lab_
-
-Presenting a comprehensive benchmark dataset designed to assess the proficiency of AI-generation detectors amidst real-world scenarios.
-Welcome to try detection via our **[online demo](https://detect.westlake.edu.cn)**!
-
-</div>
-
-## 📌 Table of Contents
-
-- [Introduction](#-introduction)
-- [Activities](#-activities)
-- [Dataset](#-dataset)
-- [Try Detection](#computer--try-detection)
-- [Data Samples](#-data-samples)
-- [Citation](#-citation)
-<!-- - [Contributing](#-contributing) -->
-
-## 🚀 Introduction
-
-Recent advances in large language models have enabled them to reach a level of text generation comparable to that of humans.
-These models show powerful capabilities across a wide range of content, including news article writing, story generation, and scientific writing.
-Such capability further narrows the gap between human-authored and machine-generated texts, highlighting the importance of machine-generated text detection to avoid potential risks such as fake news propagation and plagiarism.
-In practical scenarios, the detector faces texts from various domains or LLMs without knowing their sources.
-
-To this end, we build **a comprehensive testbed for deepfake text detection**, by gathering texts from various human writings and deepfake texts generated by different LLMs.
-This repository contains the data to testify deepfake detection methods described in our paper, [MAGE: Machine-generated Text Detection in the Wild](https://aclanthology.org/2024.acl-long.3/).
-Welcome to test your detection methods on our testbed!
-
-## 📅 Activities
-
-- 🎉 **May 16, 2024**: Our paper was accepted by ACL 2024!
-- 🎉 **June 19, 2023**: Update two 'wilder' testbeds! We go one step wilder by constructing an additional testset with texts from unseen domains generated by an unseen model, to testify the detection ability in more practical scenarios.
-  We consider four new datasets: CNN/DailyMail, DialogSum, PubMedQA and IMDb to test the detection of deepfake news, deepfake dialogues, deepfake scientific answers and deepfake movie reviews.
-  We sample 200 instances from each dataset and use a newly developed LLM, i.e., GPT-4, with specially designed prompts to create deepfake texts, establishing an "Unseen Domains & Unseen Model" scenario.
-  Previous work demonstrates that detection methods are vulnerable to being deceived by target texts.
-  Therefore, we also paraphrase each sentence individually for both human-written and machine-generated texts, forming an even more challenging testbed.
-  We adopt gpt-3.5-trubo as the zero-shot paraphraser and consider all paraphrased texts as machine-generated.
-- May 25, 2023: Initial dataset release including texts from 10 domains and 27 LLMs, contributing to 6 testbeds with increasing detection difficulty.
-
-## 📝 Dataset
-
-The dataset consists of **447,674** human-written and machine-generated texts from a wide range of sources in the wild:
-
-- Human-written texts from **10 datasets** covering a wide range of writing tasks, e.g., news article writing, story generation, scientific writing, etc.
-- Machine-generated texts generated by **27 mainstream LLMs** from 7 sources, e.g., OpenAI, LLaMA, and EleutherAI, etc.
-- **8 systematic testbed**s with increasing wildness and detection difficulty.
-
-### 📥 How to Get the Data
-
-#### 1. Huggingface
-
-You can access the full dataset, which includes the Cross-domains & Cross-models testbed and two additional wilder test sets, through the [Huggingface API](https://huggingface.co/datasets/yaful/MAGE):
-
-```python
-from datasets import load_dataset
-dataset = load_dataset("yaful/MAGE")
-```
-
-which includes traditional splits (train.csv, valid.csv and test.csv) and two wilder test sets (test_ood_set_gpt.csv and test_ood_set_gpt_para.csv).
-The csv files have three columns: text, label (0 for machine-generated and
-1 for human-written) and text source information (e.g., ''cmv_human'' denotes the text is written by humans,
-whereas ''roct_machine_continuation_flan_t5_large'' denotes the text is generated by ''flan_t5_large'' using continuation prompt).
-
-To obtain the 6 testbeds mentioned in our paper, simply apply the provided script:
-
-```shell
-python3 deployment/prepare_testbeds.py DATA_PATH
-```
-
-Replace ''DATA_PATH'' with the output data directory where you want to save the 6 testbeds.
-
-#### 2. Cloud Drive
-
-Alternatively, you can access the 6 testbeds by downloading them directly through [Google Drive](https://drive.google.com/drive/folders/1p09vDiEvoA-ZPmpqkB2WApcwMQWiiMRl?usp=sharing)
-or [Tencent Weiyun](https://share.weiyun.com/JUWQxF4H)：
-
-The folder contains 4 packages:
-
-- testbeds_processed.zip: 6 testbeds based on the ''processed'' version, which can be directly used for detecting in-distribution and out-of-distribution detection performance.
-- wilder_testsets.zip: 2 wilder test sets with texts processed, aiming for (1) detecting deepfake text generated by GPT-4, and (2) detecting deepfake text in paraphrased versions.
-- source.zip: Source texts of human-written texts and corresponding texts generated by LLMs, without filtering.
-- processed.zip: This is a refined version of the "source" that filters out low-quality texts and specifies sources as CSV file names. For example, the "cmv_machine_specified_gpt-3.5-trubo.csv" file contains texts from the CMV domain generated by the "gpt-3.5-trubo" model using specific prompts, while "cmv_human" includes human-written CMV texts.
-
-## :computer: Try Detection
-
-### Python Environment
-
-For deploying the Longformer detector or training your own detector using our data, simply install the following packages:
-
-```shell
-pip install transformers
-pip install datasets
-pip install clean-text # for data preprocessing
-```
-
-Or you can run:
-
-```shell
-pip install -r requirements.txt
-```
-
-### Model Access
-
-Our Longformer detector, which has been trained on the entire dataset, is now accessible through [Huggingface](https://huggingface.co/yaful/MAGE). Additionally, you can try detection directly using our [online demo](https://detect.westlake.edu.cn/).
-
-###
-
-We have refined the decision boundary based on out-of-distribution settings. To ensure optimal performance, we recommend preprocessing texts before sending them to the detector.
-
-```python
-import torch
-import os
-from transformers import AutoModelForSequenceClassification,AutoTokenizer
-from deployment import preprocess, detect
-
-# init
-device = 'cpu' # use 'cuda:0' if GPU is available
-# model_dir = "nealcly/detection-longformer" # model in our paper
-model_dir = "yaful/MAGE" # model in the online demo
-tokenizer = AutoTokenizer.from_pretrained(model_dir)
-model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device)
-
-text = "Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives."
-# preprocess
-text = preprocess(text)
-# detection
-result = detect(text,tokenizer,model,device)
-```
-
-### Detection Performance
-
-#### In-distribution Detection
-
-| Testbed                              | HumanRec | MachineRec | AvgRec | AUROC |
-| ------------------------------------ | -------- | ---------- | ------ | ----- |
-| White-box                            | 97.30%   | 95.91%     | 96.60% | 0.99  |
-| Arbitrary-domains & Model–specific   | 95.25%   | 96.94%     | 96.60% | 0.99  |
-| Fixed-domain & Arbitrary-models      | 89.78%   | 97.24%     | 93.51% | 0.99  |
-| Arbitrary-domains & Arbitrary-models | 82.80%   | 98.27%     | 90.53% | 0.99  |
-
-#### Out-of-distribution Detection
-
-| Testbed           | HumanRec | MachineRec | AvgRec | AUROC |
-| ----------------- | -------- | ---------- | ------ | ----- |
-| Unseen Model Sets | 83.31%   | 89.90%     | 86.61% | 0.95  |
-| Unseen Domains    | 38.05%   | 98.75%     | 68.40% | 0.93  |
-
-#### Wilder Testsets
-
-| Testbed                       | HumanRec | MachineRec | AvgRec | AUROC |
-| ----------------------------- | -------- | ---------- | ------ | ----- |
-| Unseen Domains & Unseen Model | 88.78%   | 84.12%     | 86.54% | 0.94  |
-| Paraphrase                    | 88.78%   | 37.05%     | 62.92% | 0.75  |
-
-## 📄 Data Samples
-
-All instances are stored as rows in a csv format, with each row consiting of 3 columns: _Text_, _Label_ (0 for machine-generated and 1 for human-written) and _Index_ (indexes in the original data source, used restore alignment after filtering).
-
-#### News Article
-
-| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | Label |
-| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives.                                                                                                                                                                                                                                                                                                                                                                                         | 1     |
-| Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people who applied for an Apple Card beta account last week will be sent invitations just before April 26, when everyone else can start using it too. The Apple Card is designed with privacy in mind: users aren't able to link their cards or view detailed transaction histories online as some other services do. The app itself also stores no personal data on your phone, only a virtual version in its secure payment system so you don't have to give Apple access to any sensitive details. It uses machine learning-based algorithms that learn from how you spend to provide personalized recommendations about your spending habits. Your transactions are stored offline on your device, the company says, while information like email addresses remains encrypted during transit between your devices and the cloud. And if you ever lose your physical card, Apple has said there's a way to temporarily disable the card without affecting payments until the actual one arrives. | 0     |
-| Today marks the beginning of a new era of financial technology: Apple Card is now available to all users in the United States. The long-awaited credit card from Apple, which was announced earlier this year, is now available for everyone to sign up and use. With features such as cashback on purchases and robust security measures, Apple Card could revolutionize how people make payments. This could be the start of a new wave of digital payment options.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | 0     |
-
-#### Opinion Statement
-
-| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Label |
-| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| Look, I know this is a touchy subject, and while people might say I'm young and didn't understand the complexity of wars, just hear me out. Vietnam was essentially a communist state, due to influences from China and USSR, which were alliances (the former is debatable) of Vietnam during the war. After the war, our country has suffered multiple economic depressions, and famines due to the incompetence of our liberally named Communist Party. Granted the South Vietnam government wasn't any better, but what the U.S wanted for Vietnam was for the best. I understand that, technically the US did not wage war with our people, but stood against the spread of communism in Asia, and with our strategic location, a battle surely followed. The US did not deliberately invaded our country. And look at what they did to the world. Defeated the Nazis and fascist countries, uplifted South Korea, Japan (which were both smaller and less resourceful than my country) to their respectable position on the world map today. And what had the sole communist party in my country done? Nothing but left our people in the struggle of a third-world country. And China is still brazenly harassing our borders and seas to this very day, just because our army is incapable of standing up for themselves. Please tell me if I was wrong and why the North won was a good idea. Edit: My view has changed. It was not simple as I thought it was. Generally it can be summarized into those points: involvement of China, motives and war crimes committed by the US, and there was no hope in the governing system. Communism has not helped our people a bit, but no one can really advocates for America either. We as a nation should look to develop our own path. Insights are still very much appreciated. And thanks for the discussions.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | 1     |
-| Look, I know this is a touchy subject, and while people might say I'm young and didn't understand the complexity of wars, just hear me out. Vietnam was essentially a lost war. A war where we fought against the communists, but lost, after years of fighting and thousands of lives lost. We were a technologically advanced nation, but outmatched by the communists who were determined to destroy us. And they almost did. So when I think about Iraq, I can't help but compare it to Vietnam. And the only thing I'm seeing is our forces being put in a situation where they can't win. Let's start with the weapons. I'm not a weapons expert by any means, so I don't know all the fine details. But the simple facts are this: the communists had the Russians, and we had the U.S. (and other allies). Well, the communists have just as many weapons as we have, if not more. I understand that we can win by outnumbering them, but that is very difficult. Most likely we will have to use sophisticated weapons, but then we get into the tricky area of international law. Can you really justify dropping a bomb on a country that has a pretty advanced military force (think of North Korea, for example)? The answer might be yes, because if you don't do that you're handing the war to them, but then you have to ask yourself if you really want to start that slippery slope. Now there are some people who think that if we just let the terrorists have their way with us, then we will send a message to the world. Well, if that's the case, then what about the message we send by having weapons that are supposedly sophisticated enough to kill entire countries? You can't send a message by allowing innocent people to die, and if you want to kill innocent people, then you might as well start killing people at home. So there are people who say we should use these weapons in Iraq, and there are others who say we shouldn't, and there are the people who have their own ideas. But the one thing I know is this: we are in a very difficult position. We don't have the technology to back up our claims that we are the good guys, and we don't want to lose by being outmatched, so the only thing we can do is back out of the war. But this brings up a very interesting point. I wonder if Bush, who has been preaching against the communists, is going to back out of Iraq. And if he doesn't, what kind of message does that send? I know that he wants to send a message to the rest of the world, but do we really want to send that message? If we do, then what about the message we send by supporting one of the richest nations in the world, and supporting war that many of us don't even want? I know that many of you disagree with me, and I'm sorry if this is rude, but I'm just trying to get people to think. I'm not trying to be mean, and I know that I'm not right, but at least I have something to say. I know that I can't change anything, but I know that I can at least try. | 0     |
-| It is understandable that you may wish the United States had won the Vietnam War, however, it is important to recognize that the Vietnam War was a complex conflict with many political and social implications. In reality, it is impossible to predict what would have happened if the U.S. had won the war. The war could have potentially resulted in more loss of life and suffering for the Vietnamese people. It is also important to consider that the war united the Vietnamese people and eventually led to the reunification of Vietnam in 1976, which could not have occurred if the U.S. had been victorious. Therefore, while it can be tempting to look back on history and wish for a different outcome, it is important to recognize the complexities of the Vietnam War and the positive outcomes that have come from it.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | 0     |
-
-#### Long-form Answer
-
-| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Label |
-| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| That is called bootstrap problem. How can you program something when no software exists that lets you program things. And how can a computer read what to do, if it doesn't know how to read. The answer is that you have to write a really simple program yourself, onto the hardware. It never changes for a computer, and is used every time you turn it on. That tiny program doesn't do anything except tell every part of the computer what it is and where it can get the stuff it needs. This includes really basic stuff, like storage adresses and and how to read them. From then on, the hardware can look up how to use the screen, how to read the keyboard, all those things. It's of course a bit more complicated than that, but once you have that first spark going, you can build up on that and program away.,We did use keyboards. They just weren't connected to the computer. You typed in your command on what was basically a typewriter which then"punched" the information onto cards. These were known as Hollerith Punch Cards - the machine looked like this: URL0 You then took the stack of cards very carefully to the computer hopper and fed them in. They had to stay in the same order they were punched for your program to work.', "Originally, computers were little more than some cathodes connected by cables. Programming them was done by changing the connections. A little later, computers had readers that took in cards with holes in certain distances, serving as binary input. I imagine.the first keyboard simply used that reader's interface,Well, for a pretty user-friendly way, there's the original [Altair 8800] (URL1). Note the switches on the front; when entering a program, you would set them to a position representing a number representing an operation of the computer, then press a button to store it in the memory and proceed to the next position. When you were done you'd press some other button to start the program you just "typed." Of course, all would be lost when you turned off the power.', "Via use of physical digital circuitry. I kid you not. As a project for an introductory course to digitial systems we had to make a ping pong game explicitly from digital circuits using specific IC's (and the basic NOTXORORAND gate IC's only). There was no code whatsoever in this system. Once you manage that, everything else is easy.,Punched cards preceded computers by [decades] (URL2). They could be used with sorters and counters to answer fairly sophisticated questions, as long as the data fit on the card. When business computers become useful they had the cards and the keyboards used to fill them out ready and waiting. | 1     |
-| The original programmers probably used a different way to make the computer do what they wanted it to do because keyboards did not exist yet. They might have used a different type of machine to give commands, or they might have used punch cards or other methods to write down instructions that the computer could understand. The keyboard was invented later to make it easier and faster for people to tell a computer what to do.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | 0     |
-| That is called bootstrap problem. How can you program something when no software exists that lets you program things. And how can a computer read what to do, if it has no instructions to begin with? One possible solution to the bootstrap problem is to start with a set of very basic instructions written in machine language (the language that computers speak), and use those instructions to build more complex instructions and programs. This process is also known as bootstrapping or self-hosting. For example, a programmer could start with the most basic machine instructions to add, subtract, and compare numbers, and then use those instructions to create more complex operations like conditional statements, loops, and functions. Over time, more complex programs and programming languages can be built upon this foundation. Another solution is to use a pre-existing programming language or tool to create the initial instructions or software needed to bootstrap a new system. This approach is common in the development of operating systems, where code is initially written in a higher-level language like C, and then compiled into machine code that can be run on a computer. Overall, there are several ways to approach the bootstrap problem, and the solution will typically depend on the specific goals and constraints of the project.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | 0     |
-
-#### Story Generation
-
-| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | Label |
-| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| Thomas loves science fiction, and is pleased to find himself sitting by the park entrance with Arthur C. Clarke's " Fountains of Paradise " open in his lap. He must have jogged there, he thinks to himself as he admires his brand new black-and-white Nikes. He stretches out in his black joggers and turns the page. " But there was no substitute for reality, one should beware of imitations ," he reads before shutting the book. Thomas ponders what he has read as he looks to the right; not a single car can be seen. The street appears infinite in length and the buildings fade in to the distance with it. He stands and begins his first step down the street. His movement halts when he hears a young voice behind him, " You look thirsty mister. Would you like some lemonade? " Thomas walks back past the park entrance and over to the lemonade stand, wondering how he had not noticed it before. It is beautiful, the entrance; but the park is closed now. Thomas stares up at the gates in awe. Thomas is interrupted again by the child, " 5.50, please. " Thomas looks at the counter, flustered. " I'll have the punch instead. " As the child pours the purple drink in to the cup, Thomas reaches in his pocket finding a five dollar bill and three quarters. " Keep the change ," Thomas says as he picks up his drink. Thomas sips and the sky slowly dims. He feels his breath drawn away from him as a comet sails over the park entrance. And Heaven's Gate opens. | 1     |
-| Thomas loves science fiction, and is pleased to find himself sitting by the park entrance with Arthur C. Clarke's " Fountains of Paradise " open in his lap. He must have been reading for quite a while, as it's getting dark, and the other night-time park visitors are beginning to emerge. He gets up to leave, and on his way out finds a very tiny boy walking around in circles, trying to find his parents. The little boy is quite distressed, and Thomas takes him to the park office, which is locked. Thomas finally remembers that he's got a cell phone in his pocket, and calls the number on the sign. The woman on the other end is very kind, and promises to come help the boy right away. Thomas is pleased to have been able to help, and heads off to the train station to go home. On the train, his eyes are tired, and he falls asleep. At the end of the chapter, we find out that the woman on the phone was the boy's grandmother. The boy was seven years old, and his parents had taken him to the park for a picnic. The boy had started walking around in circles when he couldn't find his mother and father again.                                                                                                                                                                                                                                                                                                                                                      | 0     |
-| Jeff was a normal guy, living a normal life. He had a family, a job, and a few friends. But above all else, he wasn't religious. He rarely thought about religion, and when he did, it was with a kind of apathy. One day, Jeff died unexpectedly. He woke up in an unfamiliar place, surrounded by people he didn't know. He was confused, but no one seemed to mind. As he looked around, Jeff noticed that everyone was dressed differently and speaking different languages. Then it hit him - he had died and gone to the afterlife. But something else struck him: none of these people were from his own religion. In fact, he didn't recognize any of the religions here. Then it dawned on him - this wasn't the afterlife of his religion, it was the afterlife of the religion whose tenets he had followed most closely, knowingly or not. He had lived his life without being religious, but had unknowingly followed a certain set of beliefs. Now, in the afterlife, he was among those who had done the same. Jeff found himself feeling strangely comforted in this new place. He realized that even though his faith had been different than others', its core values were still very much the same. This newfound understanding filled Jeff with peace and joy, and he felt like he had really come home.                                                                                                                                                                               | 0     |
-
-#### Scientific Writing
-
-| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | Label |
-| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| Although deep-learning-based methods have markedly improved the performance of speech separation over the past few years, it remains an open question how to integrate multi-channel signals for speech separation. We propose two methods, namely, early-fusion and late-fusion methods, to integrate multi-channel information based on the time-domain audio separation network, which has been proven effective in single-channel speech separation. We also propose channel-sequential-transfer learning, which is a transfer learning framework that applies the parameters trained for a lower-channel network as the initial values of a higher-channel network. For fair comparison, we evaluated our proposed methods using a spatialized version of the wsj0-2mix dataset, which is open-sourced. It was found that our proposed methods can outperform multi-channel deep clustering and improve the performance proportionally to the number of microphones. It was also proven that the performance of the late-fusion method is consistently higher than that of the single-channel method regardless of the angle difference between speakers.                                                                                    | 1     |
-| Although deep learning has achieved appealing results on several machine learning tasks, most of the models are deterministic at inference, limiting their application to single-modal settings. We propose a novel probabilistic deep learning model, namely Probabilistic Interpretation Network (PIN), which enables multi-modal inference, uncertainty quantification, and sample-based exploration by extracting latent representations from multiple modalities (e.g. vision and language) and modeling their dependencies via a probabilistic graphical model. PIN is a flexible framework that can be used to train interpretable multi-modal models as well as handle modalities in an unsupervised setting. We apply PIN to a wide variety of tasks including out-of-distribution detection, visual question answering and goal-driven dialogue. We present a new evaluation metric for goal-driven dialogue and show that PIN is capable of handling both modalities and uncertainty in this setting.                                                                                                                                                                                                                                  | 0     |
-| Although deep learning has achieved appealing results on several machine learning tasks, most of the models are deterministic at inference, limiting their application to single-modal settings. We propose a novel approach that allows to perform probabilistic inference with deep learning models. Our method is based on a variational autoencoder (VAE) and uses a mixture of Gaussians as a prior distribution for the latent variable. The VAE is trained by maximising a variational lower bound on the data log-likelihood, which can be seen as an evidence lower bound (ELBO). We introduce a novel approach to learn this ELBO, which is based on the re-parameterisation trick. This trick allows us to use standard gradient descent techniques to optimise the ELBO and consequently obtain a probabilistic latent representation for the data. We evaluate our model on a variety of datasets, including images, text, and speech. Our results show that our approach achieves comparable performance to existing deterministic models, while providing a probabilistic interpretation of the input data. Moreover, we demonstrate that our approach yields better generalisation ability when compared to deterministic models. | 0     |
-
-## 📚 Citation
-
-If you use this dataset in your research, please cite it as follows:
-
-```bibtex
-@inproceedings{li-etal-2024-mage,
-    title = "{MAGE}: Machine-generated Text Detection in the Wild",
-    author = "Li, Yafu  and
-      Li, Qintong  and
-      Cui, Leyang  and
-      Bi, Wei  and
-      Wang, Zhilin  and
-      Wang, Longyue  and
-      Yang, Linyi  and
-      Shi, Shuming  and
-      Zhang, Yue",
-    editor = "Ku, Lun-Wei  and
-      Martins, Andre  and
-      Srikumar, Vivek",
-    booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
-    month = aug,
-    year = "2024",
-    address = "Bangkok, Thailand",
-    publisher = "Association for Computational Linguistics",
-    url = "https://aclanthology.org/2024.acl-long.3",
-    doi = "10.18653/v1/2024.acl-long.3",
-    pages = "36--53",
-}
-```
-
-We welcome contributions to improve this dataset! If you have any questions or feedback, please feel free to reach out at [email protected].

src/texts/MAGE/app.py

@@ -1,74 +0,0 @@
-from transformers import pipeline
-from difflib import Differ
-from transformers import AutoModelForSequenceClassification,AutoTokenizer
-from deployment import preprocess, detect
-import gradio as gr
-
-ner_pipeline = pipeline("ner")
-
-
-def ner(text):
-    output = ner_pipeline(text)
-    output = [
-        {'entity': 'I-LOC', 'score': 0.9995369, 'index': 2, 'word': 'Chicago', 'start': 5, 'end': 12},
-        {'entity': 'I-PER', 'score': 0.99527764, 'index': 8, 'word': 'Joe', 'start': 38, 'end': 41}
-        ]
-    print(output)
-    return {"text": text, "entities": output}
-
-def diff_texts(text1, text2):
-    d = Differ()
-    return [
-        (token[2:], token[0] if token[0] != " " else None)
-        for token in d.compare(text1, text2)
-    ]
-    
-out = diff_texts(
-    "The quick brown fox jumped over the lazy dogs.",
-    "The fast brown fox jumps over lazy dogs.")
-print(out)
-
-
-def separate_characters_with_mask(text, mask):
-  """Separates characters in a string and pairs them with a mask sign.
-
-  Args:
-    text: The input string.
-
-  Returns:
-    A list of tuples, where each tuple contains a character and a mask.
-  """
-
-  return [(char, mask) for char in text]
-
-
-def detect_ai_text(text):
-    text = preprocess(text)
-    result = detect(text,tokenizer,model,device)
-    print(result)
-    output = separate_characters_with_mask(text, result)
-    return output
-
-# init
-device = 'cpu' # use 'cuda:0' if GPU is available
-# model_dir = "nealcly/detection-longformer" # model in our paper
-model_dir = "yaful/MAGE" # model in the online demo
-tokenizer = AutoTokenizer.from_pretrained(model_dir)
-model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device)
-examples = ["Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives."]
-
-demo = gr.Interface(detect_ai_text,
-             gr.Textbox(
-                 label="input text",
-                 placeholder="Enter text here...",
-                 lines=5,
-                 ),
-             gr.HighlightedText(
-                 label="AI-text detection",
-                 combine_adjacent=True,
-                 show_legend=True,
-                 color_map={"machine-generated": "red", "human-written": "green"}
-                 ),
-             examples=examples)
-
-demo.launch(share=True)

src/texts/MAGE/deployment/__init__.py

@@ -1 +0,0 @@
-from .utils import *

src/texts/MAGE/deployment/prepare_testbeds.py

@@ -1,348 +0,0 @@
-import csv
-import os
-import sys
-from collections import defaultdict
-import random
-from datasets import load_dataset
-
-set_names = [
-    "cmv",
-    "yelp",
-    "xsum",
-    "tldr",
-    "eli5",
-    "wp",
-    "roct",
-    "hswag",
-    "squad",
-    "sci_gen",
-]
-
-oai_list = [
-    # openai
-    "gpt-3.5-trubo",
-    "text-davinci-003",
-    "text-davinci-002",
-]
-llama_list = ["_7B", "_13B", "_30B", "_65B"]
-glm_list = [
-    "GLM130B",
-]
-flan_list = [
-    # flan_t5,
-    "flan_t5_small",
-    "flan_t5_base",
-    "flan_t5_large",
-    "flan_t5_xl",
-    "flan_t5_xxl",
-]
-
-opt_list = [
-    # opt,
-    "opt_125m",
-    "opt_350m",
-    "opt_1.3b",
-    "opt_2.7b",
-    "opt_6.7b",
-    "opt_13b",
-    "opt_30b",
-    "opt_iml_30b",
-    "opt_iml_max_1.3b",
-]
-bigscience_list = [
-    "bloom_7b",
-    "t0_3b",
-    "t0_11b",
-]
-eleuther_list = [
-    "gpt_j",
-    "gpt_neox",
-]
-model_sets = [
-    oai_list,
-    llama_list,
-    glm_list,
-    flan_list,
-    opt_list,
-    bigscience_list,
-    eleuther_list,
-]
-
-data_dir = sys.argv[1]
-dataset = load_dataset("yaful/DeepfakeTextDetect")
-if not os.path.exists(data_dir):
-    os.makedirs(data_dir)
-"""
-csv_path = f"{data_dir}/train.csv"
-train_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:]
-csv_path = f"{data_dir}/valid.csv"
-valid_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:]
-csv_path = f"{data_dir}/test.csv"
-test_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:]
-"""
-train_results = [
-    (row["text"], str(row["label"]), row["src"]) for row in list(dataset["train"])
-]
-valid_results = [
-    (row["text"], str(row["label"]), row["src"]) for row in list(dataset["validation"])
-]
-test_results = [
-    (row["text"], str(row["label"]), row["src"]) for row in list(dataset["test"])
-]
-merge_dict = {
-    "train": (train_results, 800),
-    "valid": (valid_results, 100),
-    "test": (test_results, 100),
-}
-
-
-test_ood_gpt = dataset["test_ood_gpt"]
-test_ood_gpt_para = dataset["test_ood_gpt_para"]
-test_ood_gpt.to_csv(os.path.join(data_dir, "test_ood_gpt.csv"))
-test_ood_gpt_para.to_csv(os.path.join(data_dir, "test_ood_gpt_para.csv"))
-
-
-# make domain-specific_model-specific (gpt_j)
-def prepare_domain_specific_model_specific():
-    tgt_model = "gpt_j"
-    testbed_dir = f"{data_dir}/domain_specific_model_specific"
-    sub_results = defaultdict(lambda: defaultdict(list))
-    print("# preparing domain-specific & model-specific ...")
-    for name in set_names:
-        print(f"## preparing {name} ...")
-        for split in ["train", "valid", "test"]:
-            split_results, split_count = merge_dict[split]
-            count = 0
-            for res in split_results:
-                info = res[2]
-                res = res[:2]
-                if name in info:
-                    # human-written
-                    if res[1] == "1" and count <= split_count:
-                        sub_results[name][split].append(res)
-                    # machine-generated
-                    if tgt_model in info:
-                        assert res[1] == "0"
-                        sub_results[name][split].append(res)
-                    count += 1
-
-        sub_dir = f"{testbed_dir}/{name}"
-        os.makedirs(sub_dir, exist_ok=True)
-        for split in ["train", "valid", "test"]:
-            print(f"{split} set: {len(sub_results[name][split])}")
-            rows = sub_results[name][split]
-            row_head = [["text", "label"]]
-            rows = row_head + rows
-            tmp_path = f"{sub_dir}/{split}.csv"
-            with open(tmp_path, "w", newline="", encoding="utf-8-sig") as f:
-                csvw = csv.writer(f)
-                csvw.writerows(rows)
-
-
-# make domain_specific_cross_models
-def prepare_domain_specific_cross_models():
-    testbed_dir = f"{data_dir}/domain_specific_cross_models"
-    sub_results = defaultdict(lambda: defaultdict(list))
-
-    print("# preparing domain_specific_cross_models ...")
-    for name in set_names:
-        print(f"## preparing {name} ...")
-        for split in ["train", "valid", "test"]:
-            split_results, split_count = merge_dict[split]
-            for res in split_results:
-                info = res[2]
-                res = res[:2]
-                if name in info:
-                    # human-written
-                    if res[1] == "1":
-                        sub_results[name][split].append(res)
-                    # machine-generated
-                    else:
-                        sub_results[name][split].append(res)
-
-        sub_dir = f"{testbed_dir}/{name}"
-        os.makedirs(sub_dir, exist_ok=True)
-        for split in ["train", "valid", "test"]:
-            print(f"{split} set: {len(sub_results[name][split])}")
-            rows = sub_results[name][split]
-            row_head = [["text", "label"]]
-            rows = row_head + rows
-            tmp_path = f"{sub_dir}/{split}.csv"
-            with open(tmp_path, "w", newline="", encoding="utf-8-sig") as f:
-                csvw = csv.writer(f)
-                csvw.writerows(rows)
-
-
-# make cross_domains_model_specific
-def prepare_cross_domains_model_specific():
-    print("# preparing cross_domains_model_specific ...")
-    for model_patterns in model_sets:
-        sub_dir = f"{data_dir}/cross_domains_model_specific/model_{model_patterns[0]}"
-        os.makedirs(sub_dir, exist_ok=True)
-        # model_pattern = dict.fromkeys(model_pattern)
-        _tmp = " ".join(model_patterns)
-        print(f"## preparing {_tmp} ...")
-
-        ood_pos_test_samples = []
-        out_split_samples = defaultdict(list)
-        for split in ["train", "valid", "test"]:
-            rows = merge_dict[split][0]
-            # print(f"Original {split} set length: {len(rows)}")
-
-            out_rows = []
-            for row in rows:
-                valid = False
-                srcinfo = row[2]
-                if row[1] == "1":  # appending all positive samples
-                    valid = True
-                for pattern in model_patterns:
-                    if pattern in srcinfo:
-                        valid = True
-                        break
-                if valid:
-                    out_rows.append(row)
-                    # out_rows.append(row+[srcinfo[0]])
-
-            out_split_samples[split] = out_rows
-
-        for split in ["train", "valid", "test"]:
-            random.seed(1)
-            rows = out_split_samples[split]
-            pos_rows = [r for r in rows if r[1] == "1"]
-            neg_rows = [r for r in rows if r[1] == "0"]
-            len_neg = len(neg_rows)
-            random.shuffle(pos_rows)
-            out_split_samples[split] = pos_rows[:len_neg] + neg_rows
-
-        for split in ["train", "valid", "test"]:
-            out_rows = [e[:-1] for e in out_split_samples[split]]
-            print(f"{split} set: {len(out_rows)} ...")
-            # xxx
-            tgt_path = f"{sub_dir}/{split}.csv"
-            with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f:
-                csvw = csv.writer(f)
-                csvw.writerows([["text", "label"]] + out_rows)
-
-
-# make cross_domains_cross_models
-def prepare_cross_domains_cross_models():
-    print("# preparing cross_domains_cross_models ...")
-    testbed_dir = f"{data_dir}/cross_domains_cross_models"
-    os.makedirs(testbed_dir, exist_ok=True)
-    for split in ["train", "valid", "test"]:
-        csv_path = f"{testbed_dir}/{split}.csv"
-
-        with open(csv_path, "w", newline="", encoding="utf-8-sig") as f:
-            rows = [row[:-1] for row in merge_dict[split][0]]
-            print(f"{split} set: {len(rows)} ...")
-            csvw = csv.writer(f)
-            csvw.writerows([["text", "label"]] + rows)
-
-
-# make unseen_models
-def prepare_unseen_models():
-    print("# preparing unseen_models ...")
-    for model_patterns in model_sets:
-        sub_dir = f"{data_dir}/unseen_models/unseen_model_{model_patterns[0]}"
-        os.makedirs(sub_dir, exist_ok=True)
-        _tmp = " ".join(model_patterns)
-        print(f"## preparing ood-models {_tmp} ...")
-
-        ood_pos_test_samples = []
-        out_split_samples = defaultdict(list)
-        for split in ["train", "valid", "test", "test_ood"]:
-            data_name = split if split != "test_ood" else "test"
-            rows = merge_dict[data_name][0]
-
-            out_rows = []
-            for row in rows:
-                valid = False
-                srcinfo = row[2]
-                for pattern in model_patterns:
-                    if split != "test_ood":
-                        if pattern in srcinfo:
-                            valid = False
-                            break
-                        valid = True
-                    else:
-                        if pattern in srcinfo:
-                            valid = True
-                            break
-                if valid:
-                    out_rows.append(row)
-
-            out_split_samples[split] = out_rows
-
-        random.seed(1)
-        test_rows = out_split_samples["test"]
-        test_pos_rows = [r for r in test_rows if r[1] == "1"]
-        test_neg_rows = [r for r in test_rows if r[1] == "0"]
-        len_aug = len(out_split_samples["test_ood"])
-        # print(len_aug)
-        random.shuffle(test_pos_rows)
-        # out_split_samples['test'] = test_pos_rows[len_aug:] + test_neg_rows
-        out_split_samples["test_ood"] = (
-            test_pos_rows[:len_aug] + out_split_samples["test_ood"]
-        )
-
-        for split in ["train", "valid", "test", "test_ood"]:
-            out_rows = [e[:-1] for e in out_split_samples[split]]
-            print(f"{split} set: {len(out_rows)}")
-
-            tgt_path = f"{sub_dir}/{split}.csv"
-            with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f:
-                csvw = csv.writer(f)
-                csvw.writerows([["text", "label"]] + out_rows)
-
-
-# make unseen_domains
-def prepare_unseen_domains():
-    print("# preparing unseen_domains ...")
-
-    testbed_dir = f"{data_dir}/unseen_domains"
-    sub_results = defaultdict(lambda: defaultdict(list))
-
-    for name in set_names:
-        sub_dir = f"{data_dir}/unseen_domains/unseen_domain_{name}"
-        os.makedirs(sub_dir, exist_ok=True)
-
-        print(f"## preparing ood-domains {name} ...")
-
-        ood_pos_test_samples = []
-        out_split_samples = defaultdict(list)
-        for split in ["train", "valid", "test", "test_ood"]:
-            data_name = split if split != "test_ood" else "test"
-            rows = merge_dict[data_name][0]
-
-            out_rows = []
-            for row in rows:
-                srcinfo = row[2]
-                valid = True if name in srcinfo else False
-                valid = not valid if split != "test_ood" else valid
-                if valid:
-                    out_rows.append(row)
-
-            out_split_samples[split] = out_rows
-
-        for split in ["train", "valid", "test", "test_ood"]:
-            out_rows = [e[:-1] for e in out_split_samples[split]]
-            print(f"{split} set: {len(out_rows)}")
-            tgt_path = f"{sub_dir}/{split}.csv"
-            with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f:
-                csvw = csv.writer(f)
-                csvw.writerows([["text", "label"]] + out_rows)
-
-
-# prepare 6 testbeds
-prepare_domain_specific_model_specific()
-print("-" * 100)
-prepare_domain_specific_cross_models()
-print("-" * 100)
-prepare_cross_domains_model_specific()
-print("-" * 100)
-prepare_cross_domains_cross_models()
-print("-" * 100)
-prepare_unseen_models()
-print("-" * 100)
-prepare_unseen_domains()
-print("-" * 100)