diff --git a/application.py b/application.py index 07730522b76c20936436b2ba979b3c6d802805c8..786fbb284a3f5d05b233ddc2fad808c8ba73cf26 100644 --- a/application.py +++ b/application.py @@ -124,13 +124,13 @@ with gr.Blocks() as demo: #url_input.change(load_image, inputs=url_input, outputs=image_view) try: - with open('example_text_real.txt','r', encoding='utf-8') as file: + with open('examples/example_text_real.txt','r', encoding='utf-8') as file: text_real_1 = file.read() - with open('example_text_real_2.txt','r', encoding='utf-8') as file: + with open('examples/example_text_real_2.txt','r', encoding='utf-8') as file: text_real_2 = file.read() - with open('example_text_LLM_topic.txt','r', encoding='utf-8') as file: + with open('examples/example_text_LLM_topic.txt','r', encoding='utf-8') as file: text_llm_topic = file.read() - with open('example_text_LLM_modification.txt','r', encoding='utf-8') as file: + with open('examples/example_text_LLM_modification.txt','r', encoding='utf-8') as file: text_llm_modification = file.read() except FileNotFoundError: print("File not found.") @@ -140,9 +140,9 @@ with gr.Blocks() as demo: title_1 = "Southampton news: Leeds target striker Cameron Archer" title_2 = "Southampton news: Leeds target striker Cameron Archer" - image_1 = "example_image_real_1.jpg.webp" - image_2 = "example_image_real_2.jpg.webp" - image_3 = "example_image_real_3.jpg" + image_1 = "examples/example_image_real_1.jpg.webp" + image_2 = "examples/example_image_real_2.jpg.webp" + image_3 = "examples/example_image_real_3.jpg" gr.Examples( examples=[ @@ -159,8 +159,4 @@ with gr.Blocks() as demo: ], ) -demo.launch(share=False) - - -# https://www.bbc.com/travel/article/20250127-one-of-the-last-traders-on-the-silk-road -# https://bbc.com/future/article/20250110-how-often-you-should-wash-your-towels-according-to-science \ No newline at end of file +demo.launch(share=False) \ No newline at end of file diff --git a/application_2.py b/application_2.py index eb606ce1025453679170253874002089fe693282..fe62315660e5586fec94442cdd254e29c6f95025 100644 --- a/application_2.py +++ b/application_2.py @@ -91,7 +91,7 @@ with gr.Blocks() as demo: with gr.Column(scale=2): with gr.Accordion("News Analysis"): detection_button = gr.Button("Verify news") - detailed_analysis = gr.HTML() + detailed_analysis = gr.HTML("
"*40) # Connect events load_button.click( @@ -116,36 +116,39 @@ with gr.Blocks() as demo: #url_input.change(load_image, inputs=url_input, outputs=image_view) try: - with open('sample_1.txt','r', encoding='utf-8') as file: - text_sample_1 = file.read() - with open('sample_2.txt','r', encoding='utf-8') as file: - text_sample_2 = file.read() - with open('sample_3.txt','r', encoding='utf-8') as file: - text_sample_3 = file.read() + with open('examples/example_text_real.txt','r', encoding='utf-8') as file: + text_real_1 = file.read() + with open('examples/example_text_real_2.txt','r', encoding='utf-8') as file: + text_real_2 = file.read() + with open('examples/example_text_LLM_topic.txt','r', encoding='utf-8') as file: + text_llm_topic = file.read() + with open('examples/example_text_LLM_modification.txt','r', encoding='utf-8') as file: + text_llm_modification = file.read() except FileNotFoundError: print("File not found.") except Exception as e: print(f"An error occurred: {e}") - title_1 = "The ancient discovery that put a Silk Road city back on the map" - title_2 = "The modern rediscovery that erased a Silk Road city from the map" + title_1 = "Southampton news: Leeds target striker Cameron Archer" + title_2 = "Southampton news: Leeds target striker Cameron Archer" - image_1 = "sample_1.jpg.webp" - image_2 = "sample_2.jpg.webp" + image_1 = "examples/example_image_real_1.jpg.webp" + image_2 = "examples/example_image_real_2.jpg.webp" + image_3 = "examples/example_image_real_3.jpg" gr.Examples( examples=[ - [title_1, image_1, text_sample_1], - [title_2, image_2, text_sample_2], - [title_1, image_2, text_sample_3], + [title_1, image_1, text_real_1 + '\n\n' + text_real_2], + [title_1, image_2, text_real_1 + '\n\n' + text_llm_modification], + [title_1, image_3, text_real_1 + '\n\n' + text_llm_topic], ], inputs=[news_title, news_image, news_content], label="Examples", example_labels=[ "2 real news", - "2 modified news", - "1 real news & 1 fake news", + "1 real news + 1 LLM modification-based news", + "1 real news + 1 LLM topic-based news", ], ) -demo.launch(share=False) +demo.launch(share=True) \ No newline at end of file diff --git a/examples/example_image_input.jpg b/examples/example_image_input.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f0eb2b3fea41dad58221043c5642af65f4206f34 Binary files /dev/null and b/examples/example_image_input.jpg differ diff --git a/example_image_real_1.jpg.webp b/examples/example_image_real_1.jpg.webp similarity index 100% rename from example_image_real_1.jpg.webp rename to examples/example_image_real_1.jpg.webp diff --git a/example_image_real_2.jpg.webp b/examples/example_image_real_2.jpg.webp similarity index 100% rename from example_image_real_2.jpg.webp rename to examples/example_image_real_2.jpg.webp diff --git a/example_image_real_3.jpg b/examples/example_image_real_3.jpg similarity index 100% rename from example_image_real_3.jpg rename to examples/example_image_real_3.jpg diff --git a/example_image_real_3.jpg.webp b/examples/example_image_real_3.jpg.webp similarity index 100% rename from example_image_real_3.jpg.webp rename to examples/example_image_real_3.jpg.webp diff --git a/example_text_LLM_modification.txt b/examples/example_text_LLM_modification.txt similarity index 100% rename from example_text_LLM_modification.txt rename to examples/example_text_LLM_modification.txt diff --git a/example_text_LLM_topic.txt b/examples/example_text_LLM_topic.txt similarity index 100% rename from example_text_LLM_topic.txt rename to examples/example_text_LLM_topic.txt diff --git a/example_text_real.txt b/examples/example_text_real.txt similarity index 100% rename from example_text_real.txt rename to examples/example_text_real.txt diff --git a/example_text_real_2.txt b/examples/example_text_real_2.txt similarity index 100% rename from example_text_real_2.txt rename to examples/example_text_real_2.txt diff --git a/src/application/content_detection.py b/src/application/content_detection.py index 6a0bad8c27b122ace83dbaf891e587c89d56fc67..eb7e7a56e50e8242c7198bffa722a61ae799046a 100644 --- a/src/application/content_detection.py +++ b/src/application/content_detection.py @@ -23,7 +23,7 @@ class NewsVerification(): self.news_prediction_label = "" self.news_prediction_score = -1 - self.found_img_url:list[str] = [] + self.found_img_url:list[str] = ["https://ichef.bbci.co.uk/ace/standard/819/cpsprodpb/8acc/live/86282470-defb-11ef-ba00-65100a906e68.jpg"] self.aligned_sentences:list[dict] = [] self.is_paraphrased:list[bool] = [] self.analyzed_table:list[list] = [] @@ -50,42 +50,61 @@ class NewsVerification(): print("\tFrom search engine:") # Classify by search engine input_sentences = split_into_sentences(self.news_text) - for sentence in input_sentences: - paraphrase, text_url, aligned_sentence, img_urls = detect_text_by_relative_search(sentence) - - text_prediction_label = "UNKNOWN" + current_index = 0 + previous_paraphrase = None + ai_sentence = { + "input_sentence": "", + "matched_sentence": "", + "label": "", + "similarity": None, + "paraphrase": False, + "url": "", + } + for index, sentence in enumerate(input_sentences): + if current_index >= index: + continue + print(f"-------index = {index}-------") + paraphrase, text_url, searched_sentences, img_urls, current_index = detect_text_by_relative_search(input_sentences, index) if paraphrase is False: - # Classify text by AI model - print("\tFrom AI model:") - text_prediction_label, text_prediction_score = detect_text_by_ai_model(sentence) - if aligned_sentence == []: - aligned_sentence = { - "input_sentence": sentence, - "matched_sentence": "", - "similarity": text_prediction_score, - "is_paraphrase_sentence": False, - "url": "", - } + # add sentence to ai_sentence + ai_sentence["input_sentence"] += sentence + if index == len(input_sentences) - 1: + # add ai_sentences to align_sentences + text_prediction_label, text_prediction_score = detect_text_by_ai_model(ai_sentence["input_sentence"]) + ai_sentence["label"] = text_prediction_label + ai_sentence["similarity"] = text_prediction_score + self.aligned_sentences.append(ai_sentence) else: - self.found_img_url.extend(img_urls) # TODO: for demo purposes - self.found_img_url.append(img_urls[0]) # TODO: for demo purposes - text_prediction_score = aligned_sentence["similarity"] - if check_human(aligned_sentence): - text_prediction_label = "HUMAN" - else: - text_prediction_label = "MACHINE" - - print(f"\ttext_prediction_label: {text_prediction_label}\n") - self.text_prediction_label.append(text_prediction_label) - self.aligned_sentences.append(aligned_sentence) - self.is_paraphrased.append(paraphrase) - self.text_referent_url.append(text_url) - self.text_prediction_score.append(text_prediction_score) - paraphrase = False - text_url = "" - aligned_sentence = {} - img_urls = [] - self.found_img_url = list(set(self.found_img_url)) + if previous_paraphrase is False or previous_paraphrase is None: + # add ai_sentences to align_sentences + if ai_sentence["input_sentence"] != "": + text_prediction_label, text_prediction_score = detect_text_by_ai_model(ai_sentence["input_sentence"]) + ai_sentence["label"] = text_prediction_label + ai_sentence["similarity"] = text_prediction_score + self.aligned_sentences.append(ai_sentence) + + # reset + ai_sentence = { + "input_sentence": "", + "matched_sentence": "", + "label": "", + "similarity": None, + "paraphrase": False, + "url": "", + } + + # add searched_sentences to align_sentences + if searched_sentences["input_sentence"] != "": + self.found_img_url.extend(img_urls) + if check_human(searched_sentences): + searched_sentences["label"] = "HUMAN" + else: + searched_sentences["label"] = "MACHINE" + + self.aligned_sentences.append(searched_sentences) + + previous_paraphrase = paraphrase + #self.found_img_url = list(set(self.found_img_url)) def detect_image_origin(self): print("CHECK IMAGE:") @@ -95,7 +114,8 @@ class NewsVerification(): self.image_referent_url = None return - print(f"\t: Img path: {self.news_image}") + for image in self.found_img_url: + print(f"\tfound_img_url: {image}") matched_url, similarity = detect_image_from_news_image(self.news_image, self.found_img_url) if matched_url is not None: print(f"matching image: {matched_url}\nsimilarity: {similarity}\n") @@ -114,6 +134,7 @@ class NewsVerification(): detected_label, score = detect_image_by_ai_model(self.news_image) if detected_label: + print(f"detected_label: {detected_label} ({score})") self.image_prediction_label = detected_label self.image_prediction_score = score self.image_referent_url = None @@ -346,13 +367,15 @@ class NewsVerification(): # short_url = self.shorten_url(self.text_referent_url[index], max_length) # source_text_url = f"""{short_url}""" - - self.text_prediction_score[index] - return f"""{input_sentence}{source_sentence}{self.text_prediction_label[index]}
({self.text_prediction_score[index]*100:.2f}%){source_text_url}""" + #label = self.aligned_sentences[index]["label"] + print(self.aligned_sentences) + print(index) + label = self.aligned_sentences[index]["label"] + score = self.aligned_sentences[index]["similarity"] + return f"""{input_sentence}{source_sentence}{label}
({score*100:.2f}%){source_text_url}""" def format_image_row(self, max_length=30): - input_image = f"""""" - print(f"self.news_image = {self.news_image}") + # input_image = f"""""" if self.image_referent_url is not None or self.image_referent_url != "": source_image = f"""""" @@ -360,6 +383,8 @@ class NewsVerification(): source_image_url = f"""{short_url}""" else: source_image = "Image not found" + source_image_url = "" + return f"""input image{source_image}{self.image_prediction_label}
({self.image_prediction_score:.2f}%){source_image_url}""" def shorten_url(self, url, max_length=30): diff --git a/src/application/image/model_detection.py b/src/application/image/model_detection.py index 93ae2df8e60243fe4dca08fadc1b7cf3b3665f42..51664f5c6febad548dfb586c05fc96a58856f0df 100644 --- a/src/application/image/model_detection.py +++ b/src/application/image/model_detection.py @@ -130,7 +130,7 @@ def image_generation_detection(image_path): image_prediction_label = "MACHINE" image_confidence = min(1, 0.5 + abs(prediction - 0.2)) result += f" with confidence = {round(image_confidence * 100, 2)}%" - image_confidence = round(image_confidence * 100, 2) + # image_confidence = round(image_confidence * 100, 2) return image_prediction_label, image_confidence diff --git a/src/application/text/helper.py b/src/application/text/helper.py index 1dfcfd9866ce4aa2bc87fd2ede507595568c0360..3a0b8bdb39c370dcd8a47acd7d07e53959935106 100644 --- a/src/application/text/helper.py +++ b/src/application/text/helper.py @@ -144,6 +144,35 @@ def extract_important_phrases(paragraph: str, keywords: list[str], phrase_length return important_phrases +def connect_consecutive_indexes(nums): + """ + Connects consecutive integers in a list. + + Args: + nums: A list of integers. + + Returns: + A list of lists, where each inner list represents a consecutive range. + """ + + if not nums: # Handle empty input + return [] + + result = [] + start = nums[0] + end = nums[0] + + for i in range(1, len(nums)): + if nums[i] == end + 1: + end = nums[i] + else: + result.append([start, end]) + start = nums[i] + end = nums[i] + + result.append([start, end]) # Add the last range + return result + """# Example usage keywords = get_keywords(paragraph) important_phrases = extract_important_phrases(paragraph, keywords) diff --git a/src/application/text/search_detection.py b/src/application/text/search_detection.py index 422866de496ac79efb6abd51980972211994fea9..31ca781c1f18870e7691bd4f5cf06bdfe07ab6ca 100644 --- a/src/application/text/search_detection.py +++ b/src/application/text/search_detection.py @@ -33,10 +33,9 @@ MIN_RATIO_PARAPHRASE_NUM = 0.7 MAX_CHAR_SIZE = 30000 -def detect_text_by_relative_search(input_text, is_support_opposite = False): - +def detect_text_by_relative_search(input_text, index, is_support_opposite = False): checked_urls = set() - searched_phrases = generate_search_phrases(input_text) + searched_phrases = generate_search_phrases(input_text[index]) for candidate in searched_phrases: search_results = search_by_google(candidate) @@ -59,15 +58,36 @@ def detect_text_by_relative_search(input_text, is_support_opposite = False): continue page_text = content.title + "\n" + content.text + print(f"page_text: {page_text}") if len(page_text) > MAX_CHAR_SIZE: print(f"\t\t\t↑↑↑ More than {MAX_CHAR_SIZE} characters") continue - is_paraphrase, aligned_sentences = check_paraphrase(input_text, page_text, url) - #if is_paraphrase: - return is_paraphrase, url, aligned_sentences, content.images + paraphrase, aligned_first_sentences = check_paraphrase(input_text[index], page_text, url) + + if paraphrase is False: + return paraphrase, url, aligned_first_sentences, content.images, index + + sub_paraphrase = True + while sub_paraphrase == True: + index += 1 + print(f"----search {index}----") + if index >= len(input_text): + break + sub_paraphrase, sub_sentences = check_paraphrase(input_text[index], page_text, url) + print(f"sub_paraphrase: {sub_paraphrase}") + print(f"sub_sentences: {sub_sentences}") + if sub_paraphrase == True: + aligned_first_sentences["input_sentence"] += sub_sentences["input_sentence"] + aligned_first_sentences["matched_sentence"] += sub_sentences["matched_sentence"] + aligned_first_sentences["similarity"] += sub_sentences["similarity"] + aligned_first_sentences["similarity"] /= 2 + + print(f"paraphrase: {paraphrase}") + print(f"aligned_first_sentences: {aligned_first_sentences}") + return paraphrase, url, aligned_first_sentences, content.images, index - return False, None, [], [] + return False, None, [], [], index def longest_common_subsequence(arr1, arr2): """ @@ -151,7 +171,7 @@ def check_sentence(input_sentence, source_sentence, min_same_sentence_len, return False -def check_paraphrase(input_text, page_text, url, verbose=False): +def check_paraphrase(input_text, page_text, url): """ Checks if the input text is paraphrased in the content at the given URL. @@ -183,7 +203,6 @@ def check_paraphrase(input_text, page_text, url, verbose=False): return is_paraphrase_text, [] #page_text = remove_punctuation(page_text) page_sentences = split_into_sentences(page_text) - if not input_sentences or not page_sentences: return is_paraphrase_text, [] @@ -193,7 +212,7 @@ def check_paraphrase(input_text, page_text, url, verbose=False): additional_sentences.append(sentence.replace(", external", "")) page_sentences.extend(additional_sentences) - min_matching_sentences = math.ceil(len(input_sentences) * MIN_RATIO_PARAPHRASE_NUM) + # min_matching_sentences = math.ceil(len(input_sentences) * MIN_RATIO_PARAPHRASE_NUM) # Encode sentences into embeddings embeddings1 = PARAPHASE_MODEL.encode(input_sentences, convert_to_tensor=True, device=DEVICE) @@ -206,18 +225,18 @@ def check_paraphrase(input_text, page_text, url, verbose=False): alignment = {} paraphrased_sentence_count = 0 for i, sentence1 in enumerate(input_sentences): - print(f"allign: {i}") max_sim_index = np.argmax(similarity_matrix[i]) max_similarity = similarity_matrix[i][max_sim_index] is_paraphrase_sentence = max_similarity > PARAPHRASE_THRESHOLD - if 0.80 > max_similarity: + if is_paraphrase_sentence is False: alignment = { "input_sentence": sentence1, "matched_sentence": "", "similarity": max_similarity, - "is_paraphrase_sentence": is_paraphrase_sentence, + "label": "", + "paraphrase": is_paraphrase_sentence, "url": "", } else: @@ -225,7 +244,8 @@ def check_paraphrase(input_text, page_text, url, verbose=False): "input_sentence": sentence1, "matched_sentence": page_sentences[max_sim_index], "similarity": max_similarity, - "is_paraphrase_sentence": is_paraphrase_sentence, + "label": "", + "paraphrase": is_paraphrase_sentence, "url": url, } @@ -234,9 +254,6 @@ def check_paraphrase(input_text, page_text, url, verbose=False): sentence1, page_sentences[max_sim_index], MIN_SAME_SENTENCE_LEN, MIN_PHRASE_SENTENCE_LEN ): is_paraphrase_text = True - if verbose: - print(f"Paraphrase found for individual sentence: {sentence1}") - print(f"Matched sentence: {page_sentences[max_sim_index]}") #alignment.append(item) paraphrased_sentence_count += 1 if is_paraphrase_sentence else 0 @@ -245,15 +262,6 @@ def check_paraphrase(input_text, page_text, url, verbose=False): is_paraphrase_text = paraphrased_sentence_count > 0 #min_matching_sentences - if verbose: - print (f"\t\tparaphrased_sentence_count: {paraphrased_sentence_count}, min_matching_sentences: {min_matching_sentences}, total_sentence_count: {len(input_sentences)}") - print(f"Minimum matching sentences required: {min_matching_sentences}") - print(f"Total input sentences: {len(input_sentences)}") - print(f"Number of matching sentences: {paraphrased_sentence_count}") - print(f"Is paraphrase: {is_paraphrase_text}") - for item in alignment: - print(item) - return is_paraphrase_text, alignment def similarity_ratio(a, b): diff --git a/src/application/url_reader.py b/src/application/url_reader.py index bba21147eed78a7defb5b6acb00efd8a5ba5640a..d90d8087aa8e70e1b5d2238b6f0bd9c156693a05 100644 --- a/src/application/url_reader.py +++ b/src/application/url_reader.py @@ -109,4 +109,11 @@ class URLReader(): except requests.exceptions.RequestException as e: print(f"\t\t↑↑↑ Error getting URL size: {e}") - return None \ No newline at end of file + return None + + +if __name__ == '__main__': + url = "https://www.bbc.com/sport/football/articles/c2d3rdy3673o" + reader = URLReader(url) + print(f"Title: {reader.title}") + print(f"Text: {reader.text}") \ No newline at end of file diff --git a/src/images/CNN_model_classifier.py b/src/images/CNN_model_classifier.py deleted file mode 100644 index f682820938139a1d3adf057a47065a1aad404c9a..0000000000000000000000000000000000000000 --- a/src/images/CNN_model_classifier.py +++ /dev/null @@ -1,63 +0,0 @@ -import argparse - -import torch.nn -import torchvision.transforms as transforms -from PIL import Image - -from .CNN.networks.resnet import resnet50 - - -def predict_cnn(image, model_path, crop=None): - model = resnet50(num_classes=1) - state_dict = torch.load(model_path, map_location="cpu") - model.load_state_dict(state_dict["model"]) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model.to(device) - model.eval() - - # Transform - if crop is not None: - trans_init = [transforms.CenterCrop(crop)] - print("Cropping to [%i]" % crop) - trans = transforms.Compose( - trans_init - + [ - transforms.ToTensor(), - transforms.Normalize( - mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225], - ), - ], - ) - - image = trans(image.convert("RGB")) - - with torch.no_grad(): - in_tens = image.unsqueeze(0) - prob = model(in_tens).sigmoid().item() - - return prob - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("-f", "--file", default="examples_realfakedir") - parser.add_argument( - "-m", - "--model_path", - type=str, - default="weights/blur_jpg_prob0.5.pth", - ) - parser.add_argument( - "-c", - "--crop", - type=int, - default=None, - help="by default, do not crop. specify crop size", - ) - - opt = parser.parse_args() - prob = predict_cnn(Image.open(opt.file), opt.model_path, crop=opt.crop) - print(f"probability of being synthetic: {prob * 100:.2f}%") diff --git a/src/images/Diffusion/Final_Report.pdf b/src/images/Diffusion/Final_Report.pdf deleted file mode 100644 index 163dc12ed21ef51dafcef13a82ab11e553431219..0000000000000000000000000000000000000000 Binary files a/src/images/Diffusion/Final_Report.pdf and /dev/null differ diff --git a/src/images/Diffusion/Pipfile b/src/images/Diffusion/Pipfile deleted file mode 100644 index d6f0932ca4bb1aca420a8e7f5bfa4b379ababb10..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/Pipfile +++ /dev/null @@ -1,29 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[[source]] -url = "https://download.pytorch.org/whl/cu121" -verify_ssl = true -name = "downloadpytorch" - -[packages] -pandas = "*" -numpy = "*" -polars = "*" -requests = "*" -img2dataset = "*" -torch = {version = "==2.1.0", index = "downloadpytorch"} -torchvision = {version = "==0.16.0", index = "downloadpytorch"} -lightning = "*" -webdataset = "*" -matplotlib = "*" -invisible-watermark = "*" -torchdata = "*" -timm = "*" - -[dev-packages] - -[requires] -python_version = "3.11" diff --git a/src/images/Diffusion/Pipfile.lock b/src/images/Diffusion/Pipfile.lock deleted file mode 100644 index ae1b65fb50ac6d8934df9663d4b24f3d35172e40..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/Pipfile.lock +++ /dev/null @@ -1,1862 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "6d3f6afdc8443ca91cb47819723377664f9f503ab96b8717efe97d1a345cdaf3" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.11" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - }, - { - "name": "downloadpytorch", - "url": "https://download.pytorch.org/whl/cu121", - "verify_ssl": true - } - ] - }, - "default": { - "aiohttp": { - "hashes": [ - "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f", - "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c", - "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af", - "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4", - "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a", - "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489", - "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213", - "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01", - "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5", - "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361", - "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26", - "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0", - "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4", - "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8", - "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1", - "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7", - "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6", - "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a", - "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd", - "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4", - "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499", - "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183", - "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544", - "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821", - "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501", - "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f", - "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe", - "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f", - "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672", - "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5", - "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2", - "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57", - "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87", - "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0", - "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f", - "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7", - "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed", - "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70", - "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0", - "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f", - "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d", - "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f", - "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d", - "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431", - "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff", - "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf", - "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83", - "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690", - "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587", - "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e", - "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb", - "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3", - "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66", - "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014", - "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35", - "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f", - "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0", - "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449", - "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23", - "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5", - "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd", - "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4", - "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b", - "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558", - "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd", - "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766", - "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a", - "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636", - "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d", - "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590", - "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e", - "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d", - "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c", - "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28", - "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065", - "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca" - ], - "version": "==3.9.1" - }, - "aiosignal": { - "hashes": [ - "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc", - "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17" - ], - "markers": "python_version >= '3.7'", - "version": "==1.3.1" - }, - "albumentations": { - "hashes": [ - "sha256:6b641d13733181d9ecdc29550e6ad580d1bfa9d25e2213a66940062f25e291bd", - "sha256:a6a38388fe546c568071e8c82f414498e86c9ed03c08b58e7a88b31cf7a244c6" - ], - "markers": "python_version >= '3.7'", - "version": "==1.3.1" - }, - "attrs": { - "hashes": [ - "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", - "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" - ], - "markers": "python_version >= '3.7'", - "version": "==23.1.0" - }, - "braceexpand": { - "hashes": [ - "sha256:91332d53de7828103dcae5773fb43bc34950b0c8160e35e0f44c4427a3b85014", - "sha256:e6e539bd20eaea53547472ff94f4fb5c3d3bf9d0a89388c4b56663aba765f705" - ], - "version": "==0.1.7" - }, - "certifi": { - "hashes": [ - "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", - "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" - ], - "markers": "python_version >= '3.6'", - "version": "==2023.11.17" - }, - "charset-normalizer": { - "hashes": [ - "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", - "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", - "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", - "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", - "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", - "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", - "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", - "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", - "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", - "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", - "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", - "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", - "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", - "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", - "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", - "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", - "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", - "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", - "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", - "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", - "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", - "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", - "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", - "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", - "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", - "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", - "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", - "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", - "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", - "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", - "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", - "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", - "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", - "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", - "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", - "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", - "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", - "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", - "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", - "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", - "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", - "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", - "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", - "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", - "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", - "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", - "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", - "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", - "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", - "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", - "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", - "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", - "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", - "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", - "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", - "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", - "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", - "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", - "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", - "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", - "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", - "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", - "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", - "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", - "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", - "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", - "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", - "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", - "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", - "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", - "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", - "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", - "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", - "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", - "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", - "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", - "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", - "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", - "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", - "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", - "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", - "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", - "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", - "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", - "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", - "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", - "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", - "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", - "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", - "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" - ], - "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.2" - }, - "click": { - "hashes": [ - "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" - ], - "markers": "python_version >= '3.7'", - "version": "==8.1.7" - }, - "contourpy": { - "hashes": [ - "sha256:0274c1cb63625972c0c007ab14dd9ba9e199c36ae1a231ce45d725cbcbfd10a8", - "sha256:0d7e03c0f9a4f90dc18d4e77e9ef4ec7b7bbb437f7f675be8e530d65ae6ef956", - "sha256:11f8d2554e52f459918f7b8e6aa20ec2a3bce35ce95c1f0ef4ba36fbda306df5", - "sha256:139d8d2e1c1dd52d78682f505e980f592ba53c9f73bd6be102233e358b401063", - "sha256:16a7380e943a6d52472096cb7ad5264ecee36ed60888e2a3d3814991a0107286", - "sha256:171f311cb758de7da13fc53af221ae47a5877be5a0843a9fe150818c51ed276a", - "sha256:18fc2b4ed8e4a8fe849d18dce4bd3c7ea637758c6343a1f2bae1e9bd4c9f4686", - "sha256:1c203f617abc0dde5792beb586f827021069fb6d403d7f4d5c2b543d87edceb9", - "sha256:1c2559d6cffc94890b0529ea7eeecc20d6fadc1539273aa27faf503eb4656d8f", - "sha256:1c88dfb9e0c77612febebb6ac69d44a8d81e3dc60f993215425b62c1161353f4", - "sha256:1e9dc350fb4c58adc64df3e0703ab076f60aac06e67d48b3848c23647ae4310e", - "sha256:247b9d16535acaa766d03037d8e8fb20866d054d3c7fbf6fd1f993f11fc60ca0", - "sha256:266270c6f6608340f6c9836a0fb9b367be61dde0c9a9a18d5ece97774105ff3e", - "sha256:34b9071c040d6fe45d9826cbbe3727d20d83f1b6110d219b83eb0e2a01d79488", - "sha256:3d7d1f8871998cdff5d2ff6a087e5e1780139abe2838e85b0b46b7ae6cc25399", - "sha256:461e3ae84cd90b30f8d533f07d87c00379644205b1d33a5ea03381edc4b69431", - "sha256:464b423bc2a009088f19bdf1f232299e8b6917963e2b7e1d277da5041f33a779", - "sha256:491b1917afdd8638a05b611a56d46587d5a632cabead889a5440f7c638bc6ed9", - "sha256:4a1b1208102be6e851f20066bf0e7a96b7d48a07c9b0cfe6d0d4545c2f6cadab", - "sha256:575bcaf957a25d1194903a10bc9f316c136c19f24e0985a2b9b5608bdf5dbfe0", - "sha256:5c6b28956b7b232ae801406e529ad7b350d3f09a4fde958dfdf3c0520cdde0dd", - "sha256:5d16edfc3fc09968e09ddffada434b3bf989bf4911535e04eada58469873e28e", - "sha256:5fd1810973a375ca0e097dee059c407913ba35723b111df75671a1976efa04bc", - "sha256:67b7f17679fa62ec82b7e3e611c43a016b887bd64fb933b3ae8638583006c6d6", - "sha256:68ce4788b7d93e47f84edd3f1f95acdcd142ae60bc0e5493bfd120683d2d4316", - "sha256:6d3364b999c62f539cd403f8123ae426da946e142312a514162adb2addd8d808", - "sha256:6e739530c662a8d6d42c37c2ed52a6f0932c2d4a3e8c1f90692ad0ce1274abe0", - "sha256:6fdd887f17c2f4572ce548461e4f96396681212d858cae7bd52ba3310bc6f00f", - "sha256:78e6ad33cf2e2e80c5dfaaa0beec3d61face0fb650557100ee36db808bfa6843", - "sha256:884c3f9d42d7218304bc74a8a7693d172685c84bd7ab2bab1ee567b769696df9", - "sha256:8d8faf05be5ec8e02a4d86f616fc2a0322ff4a4ce26c0f09d9f7fb5330a35c95", - "sha256:999c71939aad2780f003979b25ac5b8f2df651dac7b38fb8ce6c46ba5abe6ae9", - "sha256:99ad97258985328b4f207a5e777c1b44a83bfe7cf1f87b99f9c11d4ee477c4de", - "sha256:9e6c93b5b2dbcedad20a2f18ec22cae47da0d705d454308063421a3b290d9ea4", - "sha256:ab459a1cbbf18e8698399c595a01f6dcc5c138220ca3ea9e7e6126232d102bb4", - "sha256:b69303ceb2e4d4f146bf82fda78891ef7bcd80c41bf16bfca3d0d7eb545448aa", - "sha256:b7caf9b241464c404613512d5594a6e2ff0cc9cb5615c9475cc1d9b514218ae8", - "sha256:b95a225d4948b26a28c08307a60ac00fb8671b14f2047fc5476613252a129776", - "sha256:bd2f1ae63998da104f16a8b788f685e55d65760cd1929518fd94cd682bf03e41", - "sha256:be16975d94c320432657ad2402f6760990cb640c161ae6da1363051805fa8108", - "sha256:ce96dd400486e80ac7d195b2d800b03e3e6a787e2a522bfb83755938465a819e", - "sha256:dbd50d0a0539ae2e96e537553aff6d02c10ed165ef40c65b0e27e744a0f10af8", - "sha256:dd10c26b4eadae44783c45ad6655220426f971c61d9b239e6f7b16d5cdaaa727", - "sha256:ebeac59e9e1eb4b84940d076d9f9a6cec0064e241818bcb6e32124cc5c3e377a" - ], - "markers": "python_version >= '3.9'", - "version": "==1.2.0" - }, - "cycler": { - "hashes": [ - "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", - "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c" - ], - "markers": "python_version >= '3.8'", - "version": "==0.12.1" - }, - "dataclasses": { - "hashes": [ - "sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f", - "sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84" - ], - "version": "==0.6" - }, - "docker-pycreds": { - "hashes": [ - "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", - "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49" - ], - "version": "==0.4.0" - }, - "exifread-nocycle": { - "hashes": [ - "sha256:3f8d034d9b48d31e524a6f039a61ddf7679c592b5ebfccecd8771390d97a15d6", - "sha256:99966e378bc21abddfad7b253b2e120f298ecc909cbc80b31735cc6406c64031" - ], - "version": "==3.0.1" - }, - "filelock": { - "hashes": [ - "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e", - "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c" - ], - "markers": "python_version >= '3.8'", - "version": "==3.13.1" - }, - "fire": { - "hashes": [ - "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62" - ], - "version": "==0.4.0" - }, - "fonttools": { - "hashes": [ - "sha256:03ed3bda541e86725f6b4e1b94213f13ed1ae51a5a1f167028534cedea38c010", - "sha256:0dc7617d96b1e668eea9250e1c1fe62d0c78c3f69573ce7e3332cc40e6d84356", - "sha256:105099968b58a5b4cef6f3eb409db8ea8578b302a9d05e23fecba1b8b0177b5f", - "sha256:1b9e9ad2bcded9a1431afaa57c8d3c39143ac1f050862d66bddd863c515464a2", - "sha256:1f53a19dcdd5737440839b8394eeebb35da9ec8109f7926cb6456639b5b58e47", - "sha256:21e96b99878348c74aa58059b8578d7586f9519cbcdadacf56486737038aa043", - "sha256:2c980d60cd6ec1376206fe55013d166e5627ad0b149b5c81e74eaa913ab6134f", - "sha256:316cec50581e844c3ab69d7c82455b54c7cf18236b2f09e722faf665fbfcac58", - "sha256:37cd1ced6efb3dd6fe82e9f9bf92fd74ac58a5aefc284045f59ecd517a5fb9ab", - "sha256:392d0e3cc23daee910193625f7cf1b387aff9dd5b6f1a5f4a925680acb6dcbc2", - "sha256:3bdd7dfca8f6c9f4779384064027e8477ad6a037d6a327b09381f43e0247c6f3", - "sha256:43a3d267334109ff849c37cf3629476b5feb392ef1d2e464a167b83de8cd599c", - "sha256:45fa321c458ea29224067700954ec44493ae869b47e7c5485a350a149a19fb53", - "sha256:46eabddec12066829b8a1efe45ae552ba2f1796981ecf538d5f68284c354c589", - "sha256:4b9544b1346d99848ac0e9b05b5d45ee703d7562fc4c9c48cf4b781de9632e57", - "sha256:4ba17822a6681d06849078daaf6e03eccc9f467efe7c4c60280e28a78e8e5df9", - "sha256:5a17706b9cc24b27721613fe5773d93331ab7f0ecaca9955aead89c6b843d3a7", - "sha256:5cbf02cda8465b69769d07385f5d11e7bba19954e7787792f46fe679ec755ebb", - "sha256:6e441286d55fe7ec7c4fb36812bf914924813776ff514b744b510680fc2733f2", - "sha256:6eb2c54f7a07c92108daabcf02caf31df97825738db02a28270633946bcda4d0", - "sha256:777ba42b94a27bb7fb2b4082522fccfd345667c32a56011e1c3e105979af5b79", - "sha256:794de93e83297db7b4943f2431e206d8b1ea69cb3ae14638a49cc50332bf0db8", - "sha256:800e354e0c3afaeb8d9552769773d02f228e98c37b8cb03041157c3d0687cffc", - "sha256:847f3f49dd3423e5a678c098e2ba92c7f4955d4aab3044f6a507b0bb0ecb07e0", - "sha256:8717db3e4895e4820ade64ea379187738827ee60748223cb0438ef044ee208c6", - "sha256:8b07b857d4f9de3199a8c3d1b1bf2078c0f37447891ca1a8d9234106b9a27aff", - "sha256:8e1aefc2bf3c43e0f33f995f828a7bbeff4adc9393a7760b11456dbcf14388f6", - "sha256:a12dee6523c02ca78aeedd0a5e12bfa9b7b29896350edd5241542897b072ae23", - "sha256:a3c11d9687479f01eddef729aa737abcdea0a44fdaffb62a930a18892f186c9b", - "sha256:b6de2f0fcd3302fb82f94801002cb473959e998c14c24ec28234adb674aed345", - "sha256:ba299f1fbaa2a1e33210aaaf6fa816d4059e4d3cfe2ae9871368d4ab548c1c6a", - "sha256:ba6c23591427844dfb0a13658f1718489de75de6a46b64234584c0d17573162d", - "sha256:c4f4a5870e3b56788fb196da8cf30d0dfd51a76dc3b907861d018165f76ae4c2", - "sha256:cb472905da3049960e80fc1cf808231880d79727a8410e156bf3e5063a1c574f", - "sha256:cebcddbe9351b67166292b4f71ffdbfcce01ba4b07d4267824eb46b277aeb19a", - "sha256:e2277cba9f0b525e30de2a9ad3cb4219aa4bc697230c1645666b0deee9f914f0", - "sha256:e29d5f298d616a93a4c5963682dc6cc8cc09f6d89cad2c29019fc5fb3b4d9472", - "sha256:e3d24248221bd7151dfff0d88b1b5da02dccd7134bd576ce8888199827bbaa19", - "sha256:e50f794d09df0675da8d9dbd7c66bfcab2f74a708343aabcad41936d26556891", - "sha256:f22eb69996a0bd49f76bdefb30be54ce8dbb89a0d1246874d610f05c2aa2e69e", - "sha256:fb36e5f40191274a95938b40c0a1fa7f895e36935aea8709e1d6deff0b2d0d4f", - "sha256:ff6a698bdd435d24c379f6e8a54908cd9bb7dda23719084d56bf8c87709bf3bd" - ], - "markers": "python_version >= '3.8'", - "version": "==4.45.1" - }, - "frozenlist": { - "hashes": [ - "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6", - "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01", - "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251", - "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9", - "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b", - "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87", - "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf", - "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f", - "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0", - "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2", - "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b", - "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc", - "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c", - "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467", - "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9", - "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1", - "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a", - "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79", - "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167", - "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300", - "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf", - "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea", - "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2", - "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab", - "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3", - "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb", - "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087", - "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc", - "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8", - "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62", - "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f", - "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326", - "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c", - "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431", - "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963", - "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7", - "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef", - "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3", - "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956", - "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781", - "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472", - "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc", - "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839", - "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672", - "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3", - "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503", - "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d", - "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8", - "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b", - "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc", - "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f", - "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559", - "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b", - "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95", - "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb", - "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963", - "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919", - "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f", - "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3", - "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1", - "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e" - ], - "markers": "python_version >= '3.8'", - "version": "==1.4.0" - }, - "fsspec": { - "extras": [ - "http" - ], - "hashes": [ - "sha256:259d5fd5c8e756ff2ea72f42e7613c32667dc2049a4ac3d84364a7ca034acb8b", - "sha256:d6e462003e3dcdcb8c7aa84c73a228f8227e72453cd22570e2363e8844edfe7b" - ], - "markers": "python_version >= '3.7'", - "version": "==2022.11.0" - }, - "gitdb": { - "hashes": [ - "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4", - "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b" - ], - "markers": "python_version >= '3.7'", - "version": "==4.0.11" - }, - "gitpython": { - "hashes": [ - "sha256:22b126e9ffb671fdd0c129796343a02bf67bf2994b35449ffc9321aa755e18a4", - "sha256:cf14627d5a8049ffbf49915732e5eddbe8134c3bdb9d476e6182b676fc573f8a" - ], - "markers": "python_version >= '3.7'", - "version": "==3.1.40" - }, - "huggingface-hub": { - "hashes": [ - "sha256:40439632b211311f788964602bf8b0d9d6b7a2314fba4e8d67b2ce3ecea0e3fd", - "sha256:545eb3665f6ac587add946e73984148f2ea5c7877eac2e845549730570c1933a" - ], - "markers": "python_full_version >= '3.8.0'", - "version": "==0.17.3" - }, - "idna": { - "hashes": [ - "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", - "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" - ], - "markers": "python_version >= '3.5'", - "version": "==3.6" - }, - "imageio": { - "hashes": [ - "sha256:39999d05eb500089e60be467dd7d618f56e142229b44c3961c2b420eeb538d7e", - "sha256:d580d6576d0ae39c459a444a23f6f61fe72123a3df2264f5fce8c87784a4be2e" - ], - "markers": "python_version >= '3.8'", - "version": "==2.33.0" - }, - "img2dataset": { - "hashes": [ - "sha256:6cebf02977eb782ca738d8f63cc225c49aa42a0aaffd19f3f5b33cc33492217a", - "sha256:885f12d104d8c551cc46247644505a10ea8f529825d448194235f2c548eb303f" - ], - "index": "pypi", - "version": "==1.42.0" - }, - "invisible-watermark": { - "hashes": [ - "sha256:644311beed9cfe4a9a5a4a46c740f47800cef184fe2e1297f3f4542e2d992f8b" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==0.2.0" - }, - "jinja2": { - "hashes": [ - "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", - "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" - ], - "markers": "python_version >= '3.7'", - "version": "==3.1.2" - }, - "joblib": { - "hashes": [ - "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1", - "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9" - ], - "markers": "python_version >= '3.7'", - "version": "==1.3.2" - }, - "kiwisolver": { - "hashes": [ - "sha256:00bd361b903dc4bbf4eb165f24d1acbee754fce22ded24c3d56eec268658a5cf", - "sha256:040c1aebeda72197ef477a906782b5ab0d387642e93bda547336b8957c61022e", - "sha256:05703cf211d585109fcd72207a31bb170a0f22144d68298dc5e61b3c946518af", - "sha256:06f54715b7737c2fecdbf140d1afb11a33d59508a47bf11bb38ecf21dc9ab79f", - "sha256:0dc9db8e79f0036e8173c466d21ef18e1befc02de8bf8aa8dc0813a6dc8a7046", - "sha256:0f114aa76dc1b8f636d077979c0ac22e7cd8f3493abbab152f20eb8d3cda71f3", - "sha256:11863aa14a51fd6ec28688d76f1735f8f69ab1fabf388851a595d0721af042f5", - "sha256:11c7de8f692fc99816e8ac50d1d1aef4f75126eefc33ac79aac02c099fd3db71", - "sha256:11d011a7574eb3b82bcc9c1a1d35c1d7075677fdd15de527d91b46bd35e935ee", - "sha256:146d14bebb7f1dc4d5fbf74f8a6cb15ac42baadee8912eb84ac0b3b2a3dc6ac3", - "sha256:15568384086b6df3c65353820a4473575dbad192e35010f622c6ce3eebd57af9", - "sha256:19df6e621f6d8b4b9c4d45f40a66839294ff2bb235e64d2178f7522d9170ac5b", - "sha256:1b04139c4236a0f3aff534479b58f6f849a8b351e1314826c2d230849ed48985", - "sha256:210ef2c3a1f03272649aff1ef992df2e724748918c4bc2d5a90352849eb40bea", - "sha256:2270953c0d8cdab5d422bee7d2007f043473f9d2999631c86a223c9db56cbd16", - "sha256:2400873bccc260b6ae184b2b8a4fec0e4082d30648eadb7c3d9a13405d861e89", - "sha256:2a40773c71d7ccdd3798f6489aaac9eee213d566850a9533f8d26332d626b82c", - "sha256:2c5674c4e74d939b9d91dda0fae10597ac7521768fec9e399c70a1f27e2ea2d9", - "sha256:3195782b26fc03aa9c6913d5bad5aeb864bdc372924c093b0f1cebad603dd712", - "sha256:31a82d498054cac9f6d0b53d02bb85811185bcb477d4b60144f915f3b3126342", - "sha256:32d5cf40c4f7c7b3ca500f8985eb3fb3a7dfc023215e876f207956b5ea26632a", - "sha256:346f5343b9e3f00b8db8ba359350eb124b98c99efd0b408728ac6ebf38173958", - "sha256:378a214a1e3bbf5ac4a8708304318b4f890da88c9e6a07699c4ae7174c09a68d", - "sha256:39b42c68602539407884cf70d6a480a469b93b81b7701378ba5e2328660c847a", - "sha256:3a2b053a0ab7a3960c98725cfb0bf5b48ba82f64ec95fe06f1d06c99b552e130", - "sha256:3aba7311af82e335dd1e36ffff68aaca609ca6290c2cb6d821a39aa075d8e3ff", - "sha256:3cd32d6c13807e5c66a7cbb79f90b553642f296ae4518a60d8d76243b0ad2898", - "sha256:3edd2fa14e68c9be82c5b16689e8d63d89fe927e56debd6e1dbce7a26a17f81b", - "sha256:4c380469bd3f970ef677bf2bcba2b6b0b4d5c75e7a020fb863ef75084efad66f", - "sha256:4e66e81a5779b65ac21764c295087de82235597a2293d18d943f8e9e32746265", - "sha256:53abb58632235cd154176ced1ae8f0d29a6657aa1aa9decf50b899b755bc2b93", - "sha256:5794cf59533bc3f1b1c821f7206a3617999db9fbefc345360aafe2e067514929", - "sha256:59415f46a37f7f2efeec758353dd2eae1b07640d8ca0f0c42548ec4125492635", - "sha256:59ec7b7c7e1a61061850d53aaf8e93db63dce0c936db1fda2658b70e4a1be709", - "sha256:59edc41b24031bc25108e210c0def6f6c2191210492a972d585a06ff246bb79b", - "sha256:5a580c91d686376f0f7c295357595c5a026e6cbc3d77b7c36e290201e7c11ecb", - "sha256:5b94529f9b2591b7af5f3e0e730a4e0a41ea174af35a4fd067775f9bdfeee01a", - "sha256:5c7b3b3a728dc6faf3fc372ef24f21d1e3cee2ac3e9596691d746e5a536de920", - "sha256:5c90ae8c8d32e472be041e76f9d2f2dbff4d0b0be8bd4041770eddb18cf49a4e", - "sha256:5e7139af55d1688f8b960ee9ad5adafc4ac17c1c473fe07133ac092310d76544", - "sha256:5ff5cf3571589b6d13bfbfd6bcd7a3f659e42f96b5fd1c4830c4cf21d4f5ef45", - "sha256:620ced262a86244e2be10a676b646f29c34537d0d9cc8eb26c08f53d98013390", - "sha256:6512cb89e334e4700febbffaaa52761b65b4f5a3cf33f960213d5656cea36a77", - "sha256:6c08e1312a9cf1074d17b17728d3dfce2a5125b2d791527f33ffbe805200a355", - "sha256:6c3bd3cde54cafb87d74d8db50b909705c62b17c2099b8f2e25b461882e544ff", - "sha256:6ef7afcd2d281494c0a9101d5c571970708ad911d028137cd558f02b851c08b4", - "sha256:7269d9e5f1084a653d575c7ec012ff57f0c042258bf5db0954bf551c158466e7", - "sha256:72d40b33e834371fd330fb1472ca19d9b8327acb79a5821d4008391db8e29f20", - "sha256:74d1b44c6cfc897df648cc9fdaa09bc3e7679926e6f96df05775d4fb3946571c", - "sha256:74db36e14a7d1ce0986fa104f7d5637aea5c82ca6326ed0ec5694280942d1162", - "sha256:763773d53f07244148ccac5b084da5adb90bfaee39c197554f01b286cf869228", - "sha256:76c6a5964640638cdeaa0c359382e5703e9293030fe730018ca06bc2010c4437", - "sha256:76d9289ed3f7501012e05abb8358bbb129149dbd173f1f57a1bf1c22d19ab7cc", - "sha256:7931d8f1f67c4be9ba1dd9c451fb0eeca1a25b89e4d3f89e828fe12a519b782a", - "sha256:7b8b454bac16428b22560d0a1cf0a09875339cab69df61d7805bf48919415901", - "sha256:7e5bab140c309cb3a6ce373a9e71eb7e4873c70c2dda01df6820474f9889d6d4", - "sha256:83d78376d0d4fd884e2c114d0621624b73d2aba4e2788182d286309ebdeed770", - "sha256:852542f9481f4a62dbb5dd99e8ab7aedfeb8fb6342349a181d4036877410f525", - "sha256:85267bd1aa8880a9c88a8cb71e18d3d64d2751a790e6ca6c27b8ccc724bcd5ad", - "sha256:88a2df29d4724b9237fc0c6eaf2a1adae0cdc0b3e9f4d8e7dc54b16812d2d81a", - "sha256:88b9f257ca61b838b6f8094a62418421f87ac2a1069f7e896c36a7d86b5d4c29", - "sha256:8ab3919a9997ab7ef2fbbed0cc99bb28d3c13e6d4b1ad36e97e482558a91be90", - "sha256:92dea1ffe3714fa8eb6a314d2b3c773208d865a0e0d35e713ec54eea08a66250", - "sha256:9407b6a5f0d675e8a827ad8742e1d6b49d9c1a1da5d952a67d50ef5f4170b18d", - "sha256:9408acf3270c4b6baad483865191e3e582b638b1654a007c62e3efe96f09a9a3", - "sha256:955e8513d07a283056b1396e9a57ceddbd272d9252c14f154d450d227606eb54", - "sha256:9db8ea4c388fdb0f780fe91346fd438657ea602d58348753d9fb265ce1bca67f", - "sha256:9eaa8b117dc8337728e834b9c6e2611f10c79e38f65157c4c38e9400286f5cb1", - "sha256:a51a263952b1429e429ff236d2f5a21c5125437861baeed77f5e1cc2d2c7c6da", - "sha256:a6aa6315319a052b4ee378aa171959c898a6183f15c1e541821c5c59beaa0238", - "sha256:aa12042de0171fad672b6c59df69106d20d5596e4f87b5e8f76df757a7c399aa", - "sha256:aaf7be1207676ac608a50cd08f102f6742dbfc70e8d60c4db1c6897f62f71523", - "sha256:b0157420efcb803e71d1b28e2c287518b8808b7cf1ab8af36718fd0a2c453eb0", - "sha256:b3f7e75f3015df442238cca659f8baa5f42ce2a8582727981cbfa15fee0ee205", - "sha256:b9098e0049e88c6a24ff64545cdfc50807818ba6c1b739cae221bbbcbc58aad3", - "sha256:ba55dce0a9b8ff59495ddd050a0225d58bd0983d09f87cfe2b6aec4f2c1234e4", - "sha256:bb86433b1cfe686da83ce32a9d3a8dd308e85c76b60896d58f082136f10bffac", - "sha256:bbea0db94288e29afcc4c28afbf3a7ccaf2d7e027489c449cf7e8f83c6346eb9", - "sha256:bbf1d63eef84b2e8c89011b7f2235b1e0bf7dacc11cac9431fc6468e99ac77fb", - "sha256:c7940c1dc63eb37a67721b10d703247552416f719c4188c54e04334321351ced", - "sha256:c9bf3325c47b11b2e51bca0824ea217c7cd84491d8ac4eefd1e409705ef092bd", - "sha256:cdc8a402aaee9a798b50d8b827d7ecf75edc5fb35ea0f91f213ff927c15f4ff0", - "sha256:ceec1a6bc6cab1d6ff5d06592a91a692f90ec7505d6463a88a52cc0eb58545da", - "sha256:cfe6ab8da05c01ba6fbea630377b5da2cd9bcbc6338510116b01c1bc939a2c18", - "sha256:d099e745a512f7e3bbe7249ca835f4d357c586d78d79ae8f1dcd4d8adeb9bda9", - "sha256:d0ef46024e6a3d79c01ff13801cb19d0cad7fd859b15037aec74315540acc276", - "sha256:d2e5a98f0ec99beb3c10e13b387f8db39106d53993f498b295f0c914328b1333", - "sha256:da4cfb373035def307905d05041c1d06d8936452fe89d464743ae7fb8371078b", - "sha256:da802a19d6e15dffe4b0c24b38b3af68e6c1a68e6e1d8f30148c83864f3881db", - "sha256:dced8146011d2bc2e883f9bd68618b8247387f4bbec46d7392b3c3b032640126", - "sha256:dfdd7c0b105af050eb3d64997809dc21da247cf44e63dc73ff0fd20b96be55a9", - "sha256:e368f200bbc2e4f905b8e71eb38b3c04333bddaa6a2464a6355487b02bb7fb09", - "sha256:e391b1f0a8a5a10ab3b9bb6afcfd74f2175f24f8975fb87ecae700d1503cdee0", - "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec", - "sha256:e5d706eba36b4c4d5bc6c6377bb6568098765e990cfc21ee16d13963fab7b3e7", - "sha256:ec20916e7b4cbfb1f12380e46486ec4bcbaa91a9c448b97023fde0d5bbf9e4ff", - "sha256:f1d072c2eb0ad60d4c183f3fb44ac6f73fb7a8f16a2694a91f988275cbf352f9", - "sha256:f846c260f483d1fd217fe5ed7c173fb109efa6b1fc8381c8b7552c5781756192", - "sha256:f91de7223d4c7b793867797bacd1ee53bfe7359bd70d27b7b58a04efbb9436c8", - "sha256:faae4860798c31530dd184046a900e652c95513796ef51a12bc086710c2eec4d", - "sha256:fc579bf0f502e54926519451b920e875f433aceb4624a3646b3252b5caa9e0b6", - "sha256:fcc700eadbbccbf6bc1bcb9dbe0786b4b1cb91ca0dcda336eef5c2beed37b797", - "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892", - "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f" - ], - "markers": "python_version >= '3.7'", - "version": "==1.4.5" - }, - "lazy-loader": { - "hashes": [ - "sha256:1e9e76ee8631e264c62ce10006718e80b2cfc74340d17d1031e0f84af7478554", - "sha256:3b68898e34f5b2a29daaaac172c6555512d0f32074f147e2254e4a6d9d838f37" - ], - "markers": "python_version >= '3.7'", - "version": "==0.3" - }, - "lightning": { - "hashes": [ - "sha256:3b2599a8a719916cb03526e6570356809729680c6cda09391232e2aba0a4ed4b", - "sha256:f23358dedd8f5f1151475c9d95f33e4529591c992a99cb9ae89c84bca7289525" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==2.1.2" - }, - "lightning-utilities": { - "hashes": [ - "sha256:84d09b11fe9bc16c803ae5e412874748239d73ad2f3d1b90862f99ce15a03aa0", - "sha256:9e31617eccbbadc6b737a2432fd7076ff8e24957f9c63aeba2530b189e19319c" - ], - "markers": "python_version >= '3.7'", - "version": "==0.10.0" - }, - "markupsafe": { - "hashes": [ - "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", - "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", - "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", - "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", - "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c", - "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", - "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", - "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb", - "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939", - "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", - "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", - "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", - "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", - "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", - "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", - "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", - "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd", - "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", - "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", - "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", - "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", - "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", - "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", - "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", - "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", - "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007", - "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", - "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", - "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", - "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", - "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", - "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", - "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", - "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1", - "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", - "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", - "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c", - "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", - "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823", - "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", - "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", - "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", - "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", - "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", - "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", - "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", - "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", - "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", - "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", - "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", - "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", - "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", - "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", - "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", - "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", - "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", - "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", - "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc", - "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2", - "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.3" - }, - "matplotlib": { - "hashes": [ - "sha256:01a978b871b881ee76017152f1f1a0cbf6bd5f7b8ff8c96df0df1bd57d8755a1", - "sha256:03f9d160a29e0b65c0790bb07f4f45d6a181b1ac33eb1bb0dd225986450148f0", - "sha256:091275d18d942cf1ee9609c830a1bc36610607d8223b1b981c37d5c9fc3e46a4", - "sha256:09796f89fb71a0c0e1e2f4bdaf63fb2cefc84446bb963ecdeb40dfee7dfa98c7", - "sha256:0f4fc5d72b75e2c18e55eb32292659cf731d9d5b312a6eb036506304f4675630", - "sha256:172f4d0fbac3383d39164c6caafd3255ce6fa58f08fc392513a0b1d3b89c4f89", - "sha256:1b0f3b8ea0e99e233a4bcc44590f01604840d833c280ebb8fe5554fd3e6cfe8d", - "sha256:3773002da767f0a9323ba1a9b9b5d00d6257dbd2a93107233167cfb581f64717", - "sha256:46a569130ff53798ea5f50afce7406e91fdc471ca1e0e26ba976a8c734c9427a", - "sha256:4c318c1e95e2f5926fba326f68177dee364aa791d6df022ceb91b8221bd0a627", - "sha256:4e208f46cf6576a7624195aa047cb344a7f802e113bb1a06cfd4bee431de5e31", - "sha256:533b0e3b0c6768eef8cbe4b583731ce25a91ab54a22f830db2b031e83cca9213", - "sha256:5864bdd7da445e4e5e011b199bb67168cdad10b501750367c496420f2ad00843", - "sha256:5ba9cbd8ac6cf422f3102622b20f8552d601bf8837e49a3afed188d560152788", - "sha256:6f9c6976748a25e8b9be51ea028df49b8e561eed7809146da7a47dbecebab367", - "sha256:7c48d9e221b637c017232e3760ed30b4e8d5dfd081daf327e829bf2a72c731b4", - "sha256:830f00640c965c5b7f6bc32f0d4ce0c36dfe0379f7dd65b07a00c801713ec40a", - "sha256:9a5430836811b7652991939012f43d2808a2db9b64ee240387e8c43e2e5578c8", - "sha256:aa11b3c6928a1e496c1a79917d51d4cd5d04f8a2e75f21df4949eeefdf697f4b", - "sha256:b78e4f2cedf303869b782071b55fdde5987fda3038e9d09e58c91cc261b5ad18", - "sha256:b9576723858a78751d5aacd2497b8aef29ffea6d1c95981505877f7ac28215c6", - "sha256:bddfb1db89bfaa855912261c805bd0e10218923cc262b9159a49c29a7a1c1afa", - "sha256:c7d36c2209d9136cd8e02fab1c0ddc185ce79bc914c45054a9f514e44c787917", - "sha256:d1095fecf99eeb7384dabad4bf44b965f929a5f6079654b681193edf7169ec20", - "sha256:d7b1704a530395aaf73912be741c04d181f82ca78084fbd80bc737be04848331", - "sha256:d86593ccf546223eb75a39b44c32788e6f6440d13cfc4750c1c15d0fcb850b63", - "sha256:deaed9ad4da0b1aea77fe0aa0cebb9ef611c70b3177be936a95e5d01fa05094f", - "sha256:ef8345b48e95cee45ff25192ed1f4857273117917a4dcd48e3905619bcd9c9b8" - ], - "index": "pypi", - "markers": "python_version >= '3.9'", - "version": "==3.8.2" - }, - "mpmath": { - "hashes": [ - "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", - "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c" - ], - "version": "==1.3.0" - }, - "multidict": { - "hashes": [ - "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9", - "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8", - "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03", - "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710", - "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161", - "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664", - "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569", - "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067", - "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313", - "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706", - "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2", - "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636", - "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49", - "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93", - "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603", - "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0", - "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60", - "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4", - "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e", - "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1", - "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60", - "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951", - "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc", - "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe", - "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95", - "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d", - "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8", - "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed", - "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2", - "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775", - "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87", - "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c", - "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2", - "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98", - "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3", - "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe", - "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78", - "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660", - "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176", - "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e", - "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988", - "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c", - "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c", - "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0", - "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449", - "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f", - "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde", - "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5", - "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d", - "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac", - "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a", - "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9", - "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca", - "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11", - "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35", - "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063", - "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b", - "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982", - "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258", - "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1", - "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52", - "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480", - "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7", - "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461", - "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d", - "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc", - "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779", - "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a", - "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547", - "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0", - "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171", - "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf", - "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d", - "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba" - ], - "markers": "python_version >= '3.7'", - "version": "==6.0.4" - }, - "networkx": { - "hashes": [ - "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6", - "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2" - ], - "markers": "python_version >= '3.9'", - "version": "==3.2.1" - }, - "numpy": { - "hashes": [ - "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a", - "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6", - "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2", - "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79", - "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9", - "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919", - "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d", - "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060", - "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75", - "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f", - "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe", - "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167", - "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef", - "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75", - "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3", - "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7", - "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7", - "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d", - "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b", - "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186", - "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0", - "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1", - "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6", - "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e", - "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523", - "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36", - "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841", - "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818", - "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00", - "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80", - "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440", - "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210", - "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8", - "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea", - "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec", - "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841" - ], - "index": "pypi", - "markers": "python_version >= '3.9'", - "version": "==1.26.2" - }, - "opencv-python": { - "hashes": [ - "sha256:91d5f6f5209dc2635d496f6b8ca6573ecdad051a09e6b5de4c399b8e673c60da", - "sha256:9814beca408d3a0eca1bae7e3e5be68b07c17ecceb392b94170881216e09b319", - "sha256:a7aac3900fbacf55b551e7b53626c3dad4c71ce85643645c43e91fcb19045e47", - "sha256:b983197f97cfa6fcb74e1da1802c7497a6f94ed561aba6980f1f33123f904956", - "sha256:bc31f47e05447da8b3089faa0a07ffe80e114c91ce0b171e6424f9badbd1c5cd", - "sha256:c4c406bdb41eb21ea51b4e90dfbc989c002786c3f601c236a99c59a54670a394", - "sha256:cc7adbbcd1112877a39274106cb2752e04984bc01a031162952e97450d6117f6" - ], - "markers": "python_version >= '3.6'", - "version": "==4.8.1.78" - }, - "opencv-python-headless": { - "hashes": [ - "sha256:0a0f1e9f836f7d5bad1dd164694944c8761711cbdf4b36ebbd4815a8ef731079", - "sha256:2c7d45721df9801c4dcd34683a15caa0e30f38b185263fec04a6eb274bc720f0", - "sha256:382f8c7a6a14f80091284eecedd52cee4812231ee0eff1118592197b538d9252", - "sha256:3b6bd6e1132b6f5dcb3a5bfe30fc4d341a7bfb26134da349a06c9255288ded94", - "sha256:58e70d2f0915fe23e02c6e405588276c9397844a47d38b9c87fac5f7f9ba2dcc", - "sha256:bc7197b42352f6f865c302a49140b889ec7cd957dd697e2d7fc016ad0d3f28f1", - "sha256:f3a33f644249f9ce1c913eac580e4b3ef4ce7cab0a71900274708959c2feb5e3" - ], - "markers": "python_version >= '3.6'", - "version": "==4.8.1.78" - }, - "packaging": { - "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" - ], - "markers": "python_version >= '3.7'", - "version": "==23.2" - }, - "pandas": { - "hashes": [ - "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813", - "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792", - "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406", - "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373", - "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328", - "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996", - "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf", - "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6", - "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7", - "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc", - "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1", - "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23", - "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a", - "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51", - "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572", - "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31", - "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5", - "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a", - "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003", - "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d", - "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354", - "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee", - "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa", - "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0", - "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9", - "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae", - "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==1.5.3" - }, - "pathtools": { - "hashes": [ - "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0" - ], - "version": "==0.1.2" - }, - "pillow": { - "hashes": [ - "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d", - "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de", - "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616", - "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839", - "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099", - "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a", - "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219", - "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106", - "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b", - "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412", - "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b", - "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7", - "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2", - "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7", - "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14", - "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f", - "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27", - "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57", - "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262", - "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28", - "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610", - "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172", - "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273", - "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e", - "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d", - "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818", - "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f", - "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9", - "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01", - "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7", - "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651", - "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312", - "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80", - "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666", - "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061", - "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b", - "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992", - "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593", - "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4", - "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db", - "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba", - "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd", - "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e", - "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212", - "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb", - "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2", - "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34", - "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256", - "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f", - "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2", - "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38", - "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996", - "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a", - "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793" - ], - "markers": "python_version >= '3.8'", - "version": "==10.1.0" - }, - "polars": { - "hashes": [ - "sha256:21a334e18c83a259211ca6ec182498f3a89297fde9b8f75021c6881ff4411201", - "sha256:3e904d197aabf36e37fda263470eaf51ec92fb865cdea4f93947713480199303", - "sha256:6c9e597efac74f00ef9cdfd9ba8a9128ed24276916bd3d60adc6e604530e4b37", - "sha256:812dbb0cc7027fd41ce6b3eaf100b94828fc082fab026409d48792e3e7014095", - "sha256:d383fac392b08a6d5830c99f6e735a48e390c1535c8f1e67707fcaab6863ade5", - "sha256:eec0e72dce84b85c427bbf395d2e181f33e60677695b95ee2e87fed51043bdea" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==0.19.19" - }, - "promise": { - "hashes": [ - "sha256:dfd18337c523ba4b6a58801c164c1904a9d4d1b1747c7d5dbf45b693a49d93d0" - ], - "version": "==2.3" - }, - "protobuf": { - "hashes": [ - "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7", - "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c", - "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2", - "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b", - "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050", - "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9", - "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7", - "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454", - "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480", - "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469", - "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c", - "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e", - "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db", - "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905", - "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b", - "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86", - "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4", - "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402", - "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7", - "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4", - "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99", - "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee" - ], - "markers": "python_version >= '3.7'", - "version": "==3.20.3" - }, - "psutil": { - "hashes": [ - "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28", - "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017", - "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602", - "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac", - "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a", - "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9", - "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4", - "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c", - "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c", - "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c", - "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a", - "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c", - "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57", - "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a", - "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d", - "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==5.9.6" - }, - "pyarrow": { - "hashes": [ - "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1", - "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee", - "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5", - "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c", - "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75", - "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2", - "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b", - "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775", - "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610", - "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24", - "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349", - "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002", - "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649", - "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874", - "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4", - "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da", - "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a", - "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585", - "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a", - "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198", - "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52", - "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c", - "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65", - "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1", - "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03" - ], - "markers": "python_version >= '3.7'", - "version": "==10.0.1" - }, - "pyparsing": { - "hashes": [ - "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb", - "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db" - ], - "markers": "python_full_version >= '3.6.8'", - "version": "==3.1.1" - }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.2" - }, - "pytorch-lightning": { - "hashes": [ - "sha256:58e26406215f3bfafe1b9b5b4bf37c9b8cff16494f7fd60e4db1ae37461b2b28", - "sha256:8cf4a45f74e11f8ef3ad46bebb4a621eca7a990826226390767f948e86fff542" - ], - "markers": "python_version >= '3.8'", - "version": "==2.1.2" - }, - "pytz": { - "hashes": [ - "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", - "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7" - ], - "version": "==2023.3.post1" - }, - "pywavelets": { - "hashes": [ - "sha256:00d5c37775a2baa4e5e6e9df3f93e6fc700a76bd50acd3b234bb13467cc54b6b", - "sha256:05723b35191ceb7d0c0bc2898a9ff391c0f20e8ed9b75d30211464872efcac95", - "sha256:2cae4a0151e443e915905c120435e69ad410b484ce8af4839220e43a494c7c53", - "sha256:322995ea0a57c96086782f0391934f9f00123087a62ad7bef0e778491f121931", - "sha256:34d189aed544687500a2fba5b8970951a76f62f1d140cc5f9440d9b32b14b8f5", - "sha256:49aa6abf9ac941f47f7ea26a3c7dd5c8bfcf0e903dc5ec68ed105b52bfccd4e2", - "sha256:4aca65696341aa64b98bf852d6768dbb345516710a2912419d68e9d484ddd6cd", - "sha256:4d9763987b4a79917f007c1d5df0adc81adabbad3c7c0a368f4a7f12034816f3", - "sha256:51c8e9e081af40f61d194960db0f3dc0434bbd979dafcbbd6463134b3f482f37", - "sha256:526e874ba79ee3779245737a3b8540defc7e92f6cec8f13258719cc1669f8b42", - "sha256:67b65da9ef6380a48b8b53de6d8a4f83747b84b217a37944a4dcf3a53cdf308d", - "sha256:7115439f0dff291b8f81b69caff1a240695566f17c483752a49de9576c7332a4", - "sha256:7a8b58eaf946fbee002cce460d32a0e932c6d9e158aad10eea984e7f26cda15e", - "sha256:7da6c2acd7253e5d45f371bcd6c0f34d70b2f82694420afb0631130bc89e3288", - "sha256:91847ac1b658cf985a7f91ff638ba1d4a9a0544c5480ecbf8db427baf455725a", - "sha256:9c3b10f1e1b08df4d918fa238ef5e5c51c111c4f6abdfecb19c26c540cbd8187", - "sha256:aa54e6c6f2d6953f5f962eb1d1de7f9fbc5bdf06141f58c05d0d87072a05b8be", - "sha256:c857081c037552f174732d864b55d8db4845f5e2fdf0e7bfc2df675a417906f4", - "sha256:ca2e1faaea7f7ff42c771e180635e2fb165cf23c9805c4fe05f9458bcb97d093", - "sha256:d7dc392c3d3d5415b25b5c6ab3b77bb2ac2b7ff6c4d2fb81bd4633b9ac4b66f3", - "sha256:d9e25c7cabef7ccd53f5fead26ab22152fe4cb937bad7411b5d506e2b5de38f6", - "sha256:e045ee612de58e3175ae863c34072b6bf5b45b61264c1adbd75506ce31cedbb2", - "sha256:eb123f01315c0fa54e25780f3b0ce0b096bab35f6c11cacbcd4ac9915f26508a", - "sha256:f3eba7f581a723132beb213ce4b291a51306e3d2f79241a71063294a71cfa25d", - "sha256:f457d9faee286bd542c8f1921e38b8f5f54bc1949c0e349c8f1e9f8eb6d251a6" - ], - "markers": "python_version >= '3.9'", - "version": "==1.5.0" - }, - "pyyaml": { - "hashes": [ - "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", - "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", - "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", - "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", - "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", - "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", - "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", - "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", - "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", - "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", - "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", - "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", - "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", - "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", - "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", - "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", - "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", - "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", - "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", - "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", - "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", - "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", - "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", - "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", - "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", - "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", - "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", - "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", - "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", - "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", - "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", - "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", - "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", - "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", - "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", - "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", - "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", - "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", - "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", - "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", - "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", - "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", - "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", - "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", - "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", - "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", - "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", - "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", - "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", - "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" - ], - "markers": "python_version >= '3.6'", - "version": "==6.0.1" - }, - "qudida": { - "hashes": [ - "sha256:4519714c40cd0f2e6c51e1735edae8f8b19f4efe1f33be13e9d644ca5f736dd6", - "sha256:db198e2887ab0c9aa0023e565afbff41dfb76b361f85fd5e13f780d75ba18cc8" - ], - "markers": "python_full_version >= '3.5.0'", - "version": "==0.0.4" - }, - "requests": { - "hashes": [ - "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", - "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.31.0" - }, - "safetensors": { - "hashes": [ - "sha256:04157d008385bea66d12fe90844a80d4a76dc25ec5230b5bd9a630496d1b7c03", - "sha256:04dd14f53f5500eb4c4149674216ba1000670efbcf4b1b5c2643eb244e7882ea", - "sha256:097e9af2efa8778cd2f0cba451784253e62fa7cc9fc73c0744d27212f7294e25", - "sha256:0bd0afd95c1e497f520e680ea01e0397c0868a3a3030e128438cf6e9e3fcd671", - "sha256:0ddd050e01f3e843aa8c1c27bf68675b8a08e385d0045487af4d70418c3cb356", - "sha256:16d8bbb7344e39cb9d4762e85c21df94ebeb03edac923dd94bb9ed8c10eac070", - "sha256:1a45dbf03e8334d3a5dc93687d98b6dc422f5d04c7d519dac09b84a3c87dd7c6", - "sha256:1d568628e9c43ca15eb96c217da73737c9ccb07520fafd8a1eba3f2750614105", - "sha256:1faf5111c66a6ba91f85dff2e36edaaf36e6966172703159daeef330de4ddc7b", - "sha256:2297b359d91126c0f9d4fd17bae3cfa2fe3a048a6971b8db07db746ad92f850c", - "sha256:2304658e6ada81a5223225b4efe84748e760c46079bffedf7e321763cafb36c9", - "sha256:2536b11ce665834201072e9397404170f93f3be10cca9995b909f023a04501ee", - "sha256:257d59e40a1b367cb544122e7451243d65b33c3f34d822a347f4eea6fdf97fdf", - "sha256:25a043cbb59d4f75e9dd87fdf5c009dd8830105a2c57ace49b72167dd9808111", - "sha256:270b99885ec14abfd56c1d7f28ada81740a9220b4bae960c3de1c6fe84af9e4d", - "sha256:285b52a481e7ba93e29ad4ec5841ef2c4479ef0a6c633c4e2629e0508453577b", - "sha256:2b6a2814278b6660261aa9a9aae524616de9f1ec364e3716d219b6ed8f91801f", - "sha256:2d54c2f1826e790d1eb2d2512bfd0ee443f0206b423d6f27095057c7f18a0687", - "sha256:2d87d993eaefe6611a9c241a8bd364a5f1ffed5771c74840363a6c4ed8d868f6", - "sha256:2fe6926110e3d425c4b684a4379b7796fdc26ad7d16922ea1696c8e6ea7e920f", - "sha256:303d2c0415cf15a28f8d7f17379ea3c34c2b466119118a34edd9965983a1a8a6", - "sha256:313e8472197bde54e3ec54a62df184c414582979da8f3916981b6a7954910a1b", - "sha256:35803201d980efcf964b75a0a2aee97fe5e9ecc5f3ad676b38fafdfe98e0620d", - "sha256:39d36f1d88468a87c437a1bc27c502e71b6ca44c385a9117a9f9ba03a75cc9c6", - "sha256:3b0b7b2d5976fbed8a05e2bbdce5816a59e6902e9e7c7e07dc723637ed539787", - "sha256:3b30abd0cddfe959d1daedf92edcd1b445521ebf7ddefc20860ed01486b33c90", - "sha256:3c1b1d510c7aba71504ece87bf393ea82638df56303e371e5e2cf09d18977dd7", - "sha256:3cfd1ca35eacc635f0eaa894e5c5ed83ffebd0f95cac298fd430014fa7323631", - "sha256:3f6a520af7f2717c5ecba112041f2c8af1ca6480b97bf957aba81ed9642e654c", - "sha256:413e1f6ac248f7d1b755199a06635e70c3515493d3b41ba46063dec33aa2ebb7", - "sha256:4177b456c6b0c722d82429127b5beebdaf07149d265748e97e0a34ff0b3694c8", - "sha256:42c3710cec7e5c764c7999697516370bee39067de0aa089b7e2cfb97ac8c6b20", - "sha256:44e230fbbe120de564b64f63ef3a8e6ff02840fa02849d9c443d56252a1646d4", - "sha256:48901bd540f8a3c1791314bc5c8a170927bf7f6acddb75bf0a263d081a3637d4", - "sha256:53134226053e56bd56e73f7db42596e7908ed79f3c9a1016e4c1dade593ac8e5", - "sha256:573b6023a55a2f28085fc0a84e196c779b6cbef4d9e73acea14c8094fee7686f", - "sha256:5d95ea4d8b32233910734a904123bdd3979c137c461b905a5ed32511defc075f", - "sha256:5f25297148ec665f0deb8bd67e9564634d8d6841041ab5393ccfe203379ea88b", - "sha256:645b3f1138fce6e818e79d4128afa28f0657430764cc045419c1d069ff93f732", - "sha256:660ca1d8bff6c7bc7c6b30b9b32df74ef3ab668f5df42cefd7588f0d40feadcb", - "sha256:6ace9e66a40f98a216ad661245782483cf79cf56eb2b112650bb904b0baa9db5", - "sha256:6fd80f7794554091836d4d613d33a7d006e2b8d6ba014d06f97cebdfda744f64", - "sha256:780dc21eb3fd32ddd0e8c904bdb0290f2454f4ac21ae71e94f9ce72db1900a5a", - "sha256:791edc10a3c359a2f5f52d5cddab0df8a45107d91027d86c3d44e57162e5d934", - "sha256:7a8f6f679d97ea0135c7935c202feefbd042c149aa70ee759855e890c01c7814", - "sha256:7ef010e9afcb4057fb6be3d0a0cfa07aac04fe97ef73fe4a23138d8522ba7c17", - "sha256:7ff8a36e0396776d3ed9a106fc9a9d7c55d4439ca9a056a24bf66d343041d3e6", - "sha256:82571d20288c975c1b30b08deb9b1c3550f36b31191e1e81fae87669a92217d0", - "sha256:82cbb8f4d022f2e94498cbefca900698b8ded3d4f85212f47da614001ff06652", - "sha256:83c2cfbe8c6304f0891e7bb378d56f66d2148972eeb5f747cd8a2246886f0d8c", - "sha256:845be0aafabf2a60c2d482d4e93023fecffe5e5443d801d7a7741bae9de41233", - "sha256:88b4653059c903015284a9722f9a46838c654257173b279c8f6f46dbe80b612d", - "sha256:8b58ba13a9e82b4bc3fc221914f6ef237fe6c2adb13cede3ace64d1aacf49610", - "sha256:8f69903ff49cb30b9227fb5d029bea276ea20d04b06803877a420c5b1b74c689", - "sha256:8ff8e41c8037db17de0ea2a23bc684f43eaf623be7d34906fe1ac10985b8365e", - "sha256:911b48dc09e321a194def3a7431662ff4f03646832f3a8915bbf0f449b8a5fcb", - "sha256:998fbac99ca956c3a09fe07cc0b35fac26a521fa8865a690686d889f0ff4e4a6", - "sha256:9a82bc2bd7a9a0e08239bdd6d7774d64121f136add93dfa344a2f1a6d7ef35fa", - "sha256:9d16b3b2fcc6fca012c74bd01b5619c655194d3e3c13e4d4d0e446eefa39a463", - "sha256:a257de175c254d39ccd6a21341cd62eb7373b05c1e618a78096a56a857e0c316", - "sha256:a79e16222106b2f5edbca1b8185661477d8971b659a3c814cc6f15181a9b34c8", - "sha256:ae2d5a31cfb8a973a318f7c4d2cffe0bd1fe753cdf7bb41a1939d45a0a06f964", - "sha256:ae2f67f04ed0bb2e56fd380a8bd3eef03f609df53f88b6f5c7e89c08e52aae00", - "sha256:ae5497adc68669db2fed7cb2dad81e6a6106e79c9a132da3efdb6af1db1014fa", - "sha256:b287304f2b2220d51ccb51fd857761e78bcffbeabe7b0238f8dc36f2edfd9542", - "sha256:b2f8877990a72ff595507b80f4b69036a9a1986a641f8681adf3425d97d3d2a5", - "sha256:bb4cb3e37a9b961ddd68e873b29fe9ab4a081e3703412e34aedd2b7a8e9cafd9", - "sha256:bbc2ce1f5ae5143a7fb72b71fa71db6a42b4f6cf912aa3acdc6b914084778e68", - "sha256:bda3d98e2bcece388232cfc551ebf063b55bdb98f65ab54df397da30efc7dcc5", - "sha256:bdc0d039e44a727824639824090bd8869535f729878fa248addd3dc01db30eae", - "sha256:bfa2e20342b81921b98edba52f8deb68843fa9c95250739a56b52ceda5ea5c61", - "sha256:c3807ac3b16288dffebb3474b555b56fe466baa677dfc16290dcd02dca1ab228", - "sha256:c3c9f0ca510e0de95abd6424789dcbc879942a3a4e29b0dfa99d9427bf1da75c", - "sha256:c8ed5d2c04cdc1afc6b3c28d59580448ac07732c50d94c15e14670f9c473a2ce", - "sha256:cba01c6b76e01ec453933b3b3c0157c59b52881c83eaa0f7666244e71aa75fd1", - "sha256:ce7a28bc8af685a69d7e869d09d3e180a275e3281e29cf5f1c7319e231932cc7", - "sha256:d10a9f7bae608ccfdc009351f01dc3d8535ff57f9488a58a4c38e45bf954fe93", - "sha256:d3ac139377cfe71ba04573f1cda66e663b7c3e95be850e9e6c2dd4b5984bd513", - "sha256:d5b3defa74f3723a388bfde2f5d488742bc4879682bd93267c09a3bcdf8f869b", - "sha256:d784938534e255473155e4d9f276ee69eb85455b6af1292172c731409bf9adee", - "sha256:d784a98c492c751f228a4a894c3b8a092ff08b24e73b5568938c28b8c0e8f8df", - "sha256:d8a85e3e47e0d4eebfaf9a58b40aa94f977a56050cb5598ad5396a9ee7c087c6", - "sha256:d93321eea0dd7e81b283e47a1d20dee6069165cc158286316d0d06d340de8fe8", - "sha256:da52ee0dc8ba03348ffceab767bd8230842fdf78f8a996e2a16445747143a778", - "sha256:dab431699b5d45e0ca043bc580651ce9583dda594e62e245b7497adb32e99809", - "sha256:dac4bb42f8679aadc59bd91a4c5a1784a758ad49d0912995945cd674089f628e", - "sha256:e056fb9e22d118cc546107f97dc28b449d88274207dd28872bd668c86216e4f6", - "sha256:e09000b2599e1836314430f81a3884c66a5cbabdff5d9f175b5d560d4de38d78", - "sha256:e0ccb5aa0f3be2727117e5631200fbb3a5b3a2b3757545a92647d6dd8be6658f", - "sha256:e57a5ab08b0ec7a7caf30d2ac79bb30c89168431aca4f8854464bb9461686925", - "sha256:e9a7ffb1e551c6df51d267f5a751f042b183df22690f6feceac8d27364fd51d7", - "sha256:e9c80ce0001efa16066358d2dd77993adc25f5a6c61850e4ad096a2232930bce", - "sha256:eb2c1da1cc39509d1a55620a5f4d14f8911c47a89c926a96e6f4876e864375a3", - "sha256:edcf3121890b5f0616aa5a54683b1a5d2332037b970e507d6bb7841a3a596556", - "sha256:f603bdd8deac6726d39f41688ed353c532dd53935234405d79e9eb53f152fbfb", - "sha256:f8934bdfd202ebd0697040a3dff40dd77bc4c5bbf3527ede0532f5e7fb4d970f", - "sha256:fdb4adb76e21bad318210310590de61c9f4adcef77ee49b4a234f9dc48867869", - "sha256:fdb58dee173ef33634c3016c459d671ca12d11e6acf9db008261cbe58107e579" - ], - "markers": "python_version >= '3.7'", - "version": "==0.4.1" - }, - "scikit-image": { - "hashes": [ - "sha256:003ca2274ac0fac252280e7179ff986ff783407001459ddea443fe7916e38cff", - "sha256:018d734df1d2da2719087d15f679d19285fce97cd37695103deadfaef2873236", - "sha256:22318b35044cfeeb63ee60c56fc62450e5fe516228138f1d06c7a26378248a86", - "sha256:2bcb74adb0634258a67f66c2bb29978c9a3e222463e003b67ba12056c003971b", - "sha256:2c6ef454a85f569659b813ac2a93948022b0298516b757c9c6c904132be327e2", - "sha256:3663d063d8bf2fb9bdfb0ca967b9ee3b6593139c860c7abc2d2351a8a8863938", - "sha256:3b7a6c89e8d6252332121b58f50e1625c35f7d6a85489c0b6b7ee4f5155d547a", - "sha256:5071b8f6341bfb0737ab05c8ab4ac0261f9e25dbcc7b5d31e5ed230fd24a7929", - "sha256:6a92dca3d95b1301442af055e196a54b5a5128c6768b79fc0a4098f1d662dee6", - "sha256:722b970aa5da725dca55252c373b18bbea7858c1cdb406e19f9b01a4a73b30b2", - "sha256:74ec5c1d4693506842cc7c9487c89d8fc32aed064e9363def7af08b8f8cbb31d", - "sha256:95d6da2d8a44a36ae04437c76d32deb4e3c993ffc846b394b9949fd8ded73cb2", - "sha256:9e801c44a814afdadeabf4dffdffc23733e393767958b82319706f5fa3e1eaa9", - "sha256:a05ae4fe03d802587ed8974e900b943275548cde6a6807b785039d63e9a7a5ff", - "sha256:be79d7493f320a964f8fcf603121595ba82f84720de999db0fcca002266a549a", - "sha256:c472a1fb3665ec5c00423684590631d95f9afcbc97f01407d348b821880b2cb3", - "sha256:c5c378db54e61b491b9edeefff87e49fcf7fdf729bb93c777d7a5f15d36f743e", - "sha256:cf3c0c15b60ae3e557a0c7575fbd352f0c3ce0afca562febfe3ab80efbeec0e9", - "sha256:e87872f067444ee90a00dd49ca897208308645382e8a24bd3e76f301af2352cd", - "sha256:ebdbdc901bae14dab637f8d5c99f6d5cc7aaf4a3b6f4003194e003e9f688a6fc", - "sha256:f5b23908dd4d120e6aecb1ed0277563e8cbc8d6c0565bdc4c4c6475d53608452" - ], - "markers": "python_version >= '3.9'", - "version": "==0.22.0" - }, - "scikit-learn": { - "hashes": [ - "sha256:0402638c9a7c219ee52c94cbebc8fcb5eb9fe9c773717965c1f4185588ad3107", - "sha256:0ee107923a623b9f517754ea2f69ea3b62fc898a3641766cb7deb2f2ce450161", - "sha256:1215e5e58e9880b554b01187b8c9390bf4dc4692eedeaf542d3273f4785e342c", - "sha256:15e1e94cc23d04d39da797ee34236ce2375ddea158b10bee3c343647d615581d", - "sha256:18424efee518a1cde7b0b53a422cde2f6625197de6af36da0b57ec502f126157", - "sha256:1d08ada33e955c54355d909b9c06a4789a729977f165b8bae6f225ff0a60ec4a", - "sha256:3271552a5eb16f208a6f7f617b8cc6d1f137b52c8a1ef8edf547db0259b2c9fb", - "sha256:35a22e8015048c628ad099da9df5ab3004cdbf81edc75b396fd0cff8699ac58c", - "sha256:535805c2a01ccb40ca4ab7d081d771aea67e535153e35a1fd99418fcedd1648a", - "sha256:5b2de18d86f630d68fe1f87af690d451388bb186480afc719e5f770590c2ef6c", - "sha256:61a6efd384258789aa89415a410dcdb39a50e19d3d8410bd29be365bcdd512d5", - "sha256:64381066f8aa63c2710e6b56edc9f0894cc7bf59bd71b8ce5613a4559b6145e0", - "sha256:67f37d708f042a9b8d59551cf94d30431e01374e00dc2645fa186059c6c5d78b", - "sha256:6c43290337f7a4b969d207e620658372ba3c1ffb611f8bc2b6f031dc5c6d1d03", - "sha256:6fb6bc98f234fda43163ddbe36df8bcde1d13ee176c6dc9b92bb7d3fc842eb66", - "sha256:763f0ae4b79b0ff9cca0bf3716bcc9915bdacff3cebea15ec79652d1cc4fa5c9", - "sha256:785a2213086b7b1abf037aeadbbd6d67159feb3e30263434139c98425e3dcfcf", - "sha256:8db94cd8a2e038b37a80a04df8783e09caac77cbe052146432e67800e430c028", - "sha256:a19f90f95ba93c1a7f7924906d0576a84da7f3b2282ac3bfb7a08a32801add93", - "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05", - "sha256:b8692e395a03a60cd927125eef3a8e3424d86dde9b2370d544f0ea35f78a8073", - "sha256:cb06f8dce3f5ddc5dee1715a9b9f19f20d295bed8e3cd4fa51e1d050347de525", - "sha256:dc9002fc200bed597d5d34e90c752b74df516d592db162f756cc52836b38fe0e", - "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1", - "sha256:ed932ea780517b00dae7431e031faae6b49b20eb6950918eb83bd043237950e0", - "sha256:fc4144a5004a676d5022b798d9e573b05139e77f271253a4703eed295bde0433" - ], - "markers": "python_version >= '3.8'", - "version": "==1.3.2" - }, - "scipy": { - "hashes": [ - "sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c", - "sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6", - "sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8", - "sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d", - "sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97", - "sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff", - "sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993", - "sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3", - "sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd", - "sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7", - "sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446", - "sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa", - "sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937", - "sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56", - "sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd", - "sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79", - "sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4", - "sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4", - "sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710", - "sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660", - "sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41", - "sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea", - "sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65", - "sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be", - "sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec" - ], - "markers": "python_version >= '3.9'", - "version": "==1.11.4" - }, - "sentry-sdk": { - "hashes": [ - "sha256:0017fa73b8ae2d4e57fd2522ee3df30453715b29d2692142793ec5d5f90b94a6", - "sha256:8feab81de6bbf64f53279b085bd3820e3e737403b0a0d9317f73a2c3374ae359" - ], - "version": "==1.38.0" - }, - "setproctitle": { - "hashes": [ - "sha256:00e6e7adff74796ef12753ff399491b8827f84f6c77659d71bd0b35870a17d8f", - "sha256:059f4ce86f8cc92e5860abfc43a1dceb21137b26a02373618d88f6b4b86ba9b2", - "sha256:088b9efc62d5aa5d6edf6cba1cf0c81f4488b5ce1c0342a8b67ae39d64001120", - "sha256:0d3a953c50776751e80fe755a380a64cb14d61e8762bd43041ab3f8cc436092f", - "sha256:1342f4fdb37f89d3e3c1c0a59d6ddbedbde838fff5c51178a7982993d238fe4f", - "sha256:184239903bbc6b813b1a8fc86394dc6ca7d20e2ebe6f69f716bec301e4b0199d", - "sha256:195c961f54a09eb2acabbfc90c413955cf16c6e2f8caa2adbf2237d1019c7dd8", - "sha256:1f5d9027eeda64d353cf21a3ceb74bb1760bd534526c9214e19f052424b37e42", - "sha256:200620c3b15388d7f3f97e0ae26599c0c378fdf07ae9ac5a13616e933cbd2086", - "sha256:200ede6fd11233085ba9b764eb055a2a191fb4ffb950c68675ac53c874c22e20", - "sha256:21112fcd2195d48f25760f0eafa7a76510871bbb3b750219310cf88b04456ae3", - "sha256:224602f0939e6fb9d5dd881be1229d485f3257b540f8a900d4271a2c2aa4e5f4", - "sha256:287490eb90e7a0ddd22e74c89a92cc922389daa95babc833c08cf80c84c4df0a", - "sha256:2982efe7640c4835f7355fdb4da313ad37fb3b40f5c69069912f8048f77b28c8", - "sha256:2df2b67e4b1d7498632e18c56722851ba4db5d6a0c91aaf0fd395111e51cdcf4", - "sha256:2e4a8104db15d3462e29d9946f26bed817a5b1d7a47eabca2d9dc2b995991503", - "sha256:2e71f6365744bf53714e8bd2522b3c9c1d83f52ffa6324bd7cbb4da707312cd8", - "sha256:334f7ed39895d692f753a443102dd5fed180c571eb6a48b2a5b7f5b3564908c8", - "sha256:33c5609ad51cd99d388e55651b19148ea99727516132fb44680e1f28dd0d1de9", - "sha256:37a62cbe16d4c6294e84670b59cf7adcc73faafe6af07f8cb9adaf1f0e775b19", - "sha256:38ae9a02766dad331deb06855fb7a6ca15daea333b3967e214de12cfae8f0ef5", - "sha256:38da436a0aaace9add67b999eb6abe4b84397edf4a78ec28f264e5b4c9d53cd5", - "sha256:415bfcfd01d1fbf5cbd75004599ef167a533395955305f42220a585f64036081", - "sha256:417de6b2e214e837827067048f61841f5d7fc27926f2e43954567094051aff18", - "sha256:477d3da48e216d7fc04bddab67b0dcde633e19f484a146fd2a34bb0e9dbb4a1e", - "sha256:4a6ba2494a6449b1f477bd3e67935c2b7b0274f2f6dcd0f7c6aceae10c6c6ba3", - "sha256:4fe1c49486109f72d502f8be569972e27f385fe632bd8895f4730df3c87d5ac8", - "sha256:507e8dc2891021350eaea40a44ddd887c9f006e6b599af8d64a505c0f718f170", - "sha256:53bc0d2358507596c22b02db079618451f3bd720755d88e3cccd840bafb4c41c", - "sha256:554eae5a5b28f02705b83a230e9d163d645c9a08914c0ad921df363a07cf39b1", - "sha256:59335d000c6250c35989394661eb6287187854e94ac79ea22315469ee4f4c244", - "sha256:5a740f05d0968a5a17da3d676ce6afefebeeeb5ce137510901bf6306ba8ee002", - "sha256:5bc94cf128676e8fac6503b37763adb378e2b6be1249d207630f83fc325d9b11", - "sha256:64286f8a995f2cd934082b398fc63fca7d5ffe31f0e27e75b3ca6b4efda4e353", - "sha256:664698ae0013f986118064b6676d7dcd28fefd0d7d5a5ae9497cbc10cba48fa5", - "sha256:68f960bc22d8d8e4ac886d1e2e21ccbd283adcf3c43136161c1ba0fa509088e0", - "sha256:69d565d20efe527bd8a9b92e7f299ae5e73b6c0470f3719bd66f3cd821e0d5bd", - "sha256:6a143b31d758296dc2f440175f6c8e0b5301ced3b0f477b84ca43cdcf7f2f476", - "sha256:6a249415f5bb88b5e9e8c4db47f609e0bf0e20a75e8d744ea787f3092ba1f2d0", - "sha256:6b9e62ddb3db4b5205c0321dd69a406d8af9ee1693529d144e86bd43bcb4b6c0", - "sha256:7f1d36a1e15a46e8ede4e953abb104fdbc0845a266ec0e99cc0492a4364f8c44", - "sha256:816330675e3504ae4d9a2185c46b573105d2310c20b19ea2b4596a9460a4f674", - "sha256:87e668f9561fd3a457ba189edfc9e37709261287b52293c115ae3487a24b92f6", - "sha256:897a73208da48db41e687225f355ce993167079eda1260ba5e13c4e53be7f754", - "sha256:8c331e91a14ba4076f88c29c777ad6b58639530ed5b24b5564b5ed2fd7a95452", - "sha256:950f6476d56ff7817a8fed4ab207727fc5260af83481b2a4b125f32844df513a", - "sha256:9617b676b95adb412bb69645d5b077d664b6882bb0d37bfdafbbb1b999568d85", - "sha256:9e3b99b338598de0bd6b2643bf8c343cf5ff70db3627af3ca427a5e1a1a90dd9", - "sha256:a1fcac43918b836ace25f69b1dca8c9395253ad8152b625064415b1d2f9be4fb", - "sha256:a680d62c399fa4b44899094027ec9a1bdaf6f31c650e44183b50d4c4d0ccc085", - "sha256:a6d50252377db62d6a0bb82cc898089916457f2db2041e1d03ce7fadd4a07381", - "sha256:a83ca086fbb017f0d87f240a8f9bbcf0809f3b754ee01cec928fff926542c450", - "sha256:a911b26264dbe9e8066c7531c0591cfab27b464459c74385b276fe487ca91c12", - "sha256:ab2900d111e93aff5df9fddc64cf51ca4ef2c9f98702ce26524f1acc5a786ae7", - "sha256:ab92e51cd4a218208efee4c6d37db7368fdf182f6e7ff148fb295ecddf264287", - "sha256:accb66d7b3ccb00d5cd11d8c6e07055a4568a24c95cf86109894dcc0c134cc89", - "sha256:ad6d20f9541f5f6ac63df553b6d7a04f313947f550eab6a61aa758b45f0d5657", - "sha256:aeaa71fb9568ebe9b911ddb490c644fbd2006e8c940f21cb9a1e9425bd709574", - "sha256:af2c67ae4c795d1674a8d3ac1988676fa306bcfa1e23fddb5e0bd5f5635309ca", - "sha256:af4061f67fd7ec01624c5e3c21f6b7af2ef0e6bab7fbb43f209e6506c9ce0092", - "sha256:b1067647ac7aba0b44b591936118a22847bda3c507b0a42d74272256a7a798e9", - "sha256:b5901a31012a40ec913265b64e48c2a4059278d9f4e6be628441482dd13fb8b5", - "sha256:bbbd6c7de0771c84b4aa30e70b409565eb1fc13627a723ca6be774ed6b9d9fa3", - "sha256:bdfd7254745bb737ca1384dee57e6523651892f0ea2a7344490e9caefcc35e64", - "sha256:c05ac48ef16ee013b8a326c63e4610e2430dbec037ec5c5b58fcced550382b74", - "sha256:c1c84beab776b0becaa368254801e57692ed749d935469ac10e2b9b825dbdd8e", - "sha256:c32c41ace41f344d317399efff4cffb133e709cec2ef09c99e7a13e9f3b9483c", - "sha256:c3ba57029c9c50ecaf0c92bb127224cc2ea9fda057b5d99d3f348c9ec2855ad3", - "sha256:c7951820b77abe03d88b114b998867c0f99da03859e5ab2623d94690848d3e45", - "sha256:c913e151e7ea01567837ff037a23ca8740192880198b7fbb90b16d181607caae", - "sha256:c9a402881ec269d0cc9c354b149fc29f9ec1a1939a777f1c858cdb09c7a261df", - "sha256:cbf16381c7bf7f963b58fb4daaa65684e10966ee14d26f5cc90f07049bfd8c1e", - "sha256:d4460795a8a7a391e3567b902ec5bdf6c60a47d791c3b1d27080fc203d11c9dc", - "sha256:d7f27e0268af2d7503386e0e6be87fb9b6657afd96f5726b733837121146750d", - "sha256:d876d355c53d975c2ef9c4f2487c8f83dad6aeaaee1b6571453cb0ee992f55f6", - "sha256:da0d57edd4c95bf221b2ebbaa061e65b1788f1544977288bdf95831b6e44e44d", - "sha256:ddedd300cd690a3b06e7eac90ed4452348b1348635777ce23d460d913b5b63c3", - "sha256:df3f4274b80709d8bcab2f9a862973d453b308b97a0b423a501bcd93582852e3", - "sha256:e18b7bd0898398cc97ce2dfc83bb192a13a087ef6b2d5a8a36460311cb09e775", - "sha256:e5119a211c2e98ff18b9908ba62a3bd0e3fabb02a29277a7232a6fb4b2560aa0", - "sha256:e5e08e232b78ba3ac6bc0d23ce9e2bee8fad2be391b7e2da834fc9a45129eb87", - "sha256:eae8988e78192fd1a3245a6f4f382390b61bce6cfcc93f3809726e4c885fa68d", - "sha256:f05e66746bf9fe6a3397ec246fe481096664a9c97eb3fea6004735a4daf867fd", - "sha256:f1da82c3e11284da4fcbf54957dafbf0655d2389cd3d54e4eaba636faf6d117a", - "sha256:f38d48abc121263f3b62943f84cbaede05749047e428409c2c199664feb6abc7", - "sha256:f5e7266498cd31a4572378c61920af9f6b4676a73c299fce8ba93afd694f8ae7", - "sha256:fc74e84fdfa96821580fb5e9c0b0777c1c4779434ce16d3d62a9c4d8c710df39", - "sha256:ff814dea1e5c492a4980e3e7d094286077054e7ea116cbeda138819db194b2cd" - ], - "markers": "python_version >= '3.7'", - "version": "==1.3.3" - }, - "setuptools": { - "hashes": [ - "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2", - "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6" - ], - "markers": "python_version >= '3.8'", - "version": "==69.0.2" - }, - "shortuuid": { - "hashes": [ - "sha256:27ea8f28b1bd0bf8f15057a3ece57275d2059d2b0bb02854f02189962c13b6aa", - "sha256:fc75f2615914815a8e4cb1501b3a513745cb66ef0fd5fc6fb9f8c3fa3481f789" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.11" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, - "smmap": { - "hashes": [ - "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62", - "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da" - ], - "markers": "python_version >= '3.7'", - "version": "==5.0.1" - }, - "sympy": { - "hashes": [ - "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5", - "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8" - ], - "markers": "python_version >= '3.8'", - "version": "==1.12" - }, - "termcolor": { - "hashes": [ - "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63", - "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a" - ], - "markers": "python_version >= '3.8'", - "version": "==2.4.0" - }, - "threadpoolctl": { - "hashes": [ - "sha256:2b7818516e423bdaebb97c723f86a7c6b0a83d3f3b0970328d66f4d9104dc032", - "sha256:c96a0ba3bdddeaca37dc4cc7344aafad41cdb8c313f74fdfe387a867bba93355" - ], - "markers": "python_version >= '3.8'", - "version": "==3.2.0" - }, - "tifffile": { - "hashes": [ - "sha256:1de47fa945fddaade256e25ad4f375ae65547f3c1354063aded881c32a64cf89", - "sha256:67e355e4595aab397f8405d04afe1b4ae7c6f62a44e22d933fee1a571a48c7ae" - ], - "markers": "python_version >= '3.9'", - "version": "==2023.9.26" - }, - "timm": { - "hashes": [ - "sha256:2a828afac5b710a80ec66d0f85807e171e342faf5c0703b33102d8aa206f19dc", - "sha256:9121d1cf320f7f32490d893340fd33117bda0a0270eb8282dfd52ae5fd3e1af6" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==0.9.12" - }, - "torch": { - "hashes": [ - "sha256:0d4e8c52a1fcf5ed6cfc256d9a370fcf4360958fc79d0b08a51d55e70914df46", - "sha256:3b7c6dd1ab12a9c70b29bf1ea34fcf2c519233c58c619c1a553d328955c8a602", - "sha256:4c83190ad649c77adaf6e1c616998f10598db696912ea7a410831632890b49bf", - "sha256:6ee083ba804e863af059ea284c1678c1b0628699fb0014c8e043ceed7d4ce930", - "sha256:94b60ae7562ae732554ae8744123b33d46e659c3251a5a58c7269c12e838868b", - "sha256:aa984599c2c4ffbc57c48d0d965cbe832e610c967e8179d4ac0a582c733fe112", - "sha256:fb808a620951b8cfb4b55cbaf8ace4b7a6d51c5be03d46513d73d009f43aafeb", - "sha256:ff0ee0b7ab3d6cfbf7875c8b1d130309ee5a18fbf2fda11f3da86a783e6e679c" - ], - "index": "downloadpytorch", - "version": "==2.1.0+cu121" - }, - "torchdata": { - "hashes": [ - "sha256:042db39edbc961c50a36c45b89aea4b099858140c13746c7cc7a87b1cc219d0c", - "sha256:293e399f3988fcd8d24156188342e9265273787dc0a29b1b37891a1045eeaece", - "sha256:2d2c8482313dd52652caff99dc530433d898a12bb80bc33a0a4d1680d63272e0", - "sha256:36c591d0910ede6a496d4fccd389f27e7bccabf8b6a8722712ecf28b98bda8ae", - "sha256:432295d9d33a7497d3c4aee667998af5bd9dcf55bd10b77c6af1ac72249efe22", - "sha256:53ef621460e2bc014069c126cbdcd325bf73d78836155a350767fe5f8ca29f11", - "sha256:7460a5fa298e7cd5cef98e8e6455d481e5c73d39a462a89a38918389c8153e20", - "sha256:8ac74fc6ce8bf289b8d99ea183f78e1bf2a4754ea6a2b1dcb219095b0aaacb78", - "sha256:8bbaecdbc1a7dc4a9d7dc7545ea4c12be92a3df8d2494f089e974e25591f514a", - "sha256:91a78c677a3e4e2447d888587f7ea0b4ddde81ca95adf709ba0d3dc9a4e9542d", - "sha256:94ce50572550010db0431283b5d228f8727f779aafd9cbbbcdc37028a5085603", - "sha256:9f9476a26987d90fa3f87cb09ec82b78ce6031ddcaa91851c9fa9f732a987ab8", - "sha256:ba802128f50bfa227be107027e0230581b3a4ac70d14782b44662b7c71159cf1", - "sha256:c1feae257e55b2942459a26f5597088e5daefabbd47a562081c01c0841a88e18", - "sha256:d256535648dfb94d1226f233768c6798d1841edfdbf0a09b2115e6cbbda614f9", - "sha256:eed10b1f9265b30161a8cd129a96cfc7da202cfc70acf8b6a21fd29e18274ca3", - "sha256:fa325d628aa6125c6b46b6fa27c94150ca9276edbba1042d3eb3cd9c1039b5a9" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==0.7.1" - }, - "torchmetrics": { - "hashes": [ - "sha256:217387738f84939c39b534b20d4983e737cc448d27aaa5340e0327948d97ca3e", - "sha256:fe03a8c53d0ae5800d34ea615f56295fda281282cd83f647d2184e81c1d4efee" - ], - "markers": "python_version >= '3.8'", - "version": "==1.2.1" - }, - "torchvision": { - "hashes": [ - "sha256:09dea0b374be56df4ae148e83221f172a8a6c999475e9483037ab6efa3cd6b80", - "sha256:5ea8bba7e82bf89467e7f8790499e3c20c97c7d0ccd778789ccdd2d84c647c9a", - "sha256:7a325270c7806571ceddbd27c8ece5c163cceb476f09dcca7eb5157073216b22", - "sha256:7bb32e2c163766045693e557b61f278eb54960e7cab7c5d6037b89847605e26c", - "sha256:967c0f8ada2abeed430b33e83ce7b3de5233675c8b020c18c1255dca065b4add", - "sha256:d4be401c6a9b2af7ea476e79a6397a77a952b6a3bec6218452be13d1b18b9c54", - "sha256:e0b29e67f0f2b059fc99d0bb9dae3f107a0bcd656e0310504758569a137f874b", - "sha256:e76e78d0ad43636c9884b3084ffaea8a8b61f21129fbfa456a5fe734f0affea9" - ], - "index": "downloadpytorch", - "version": "==0.16.0+cu121" - }, - "tqdm": { - "hashes": [ - "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386", - "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7" - ], - "markers": "python_version >= '3.7'", - "version": "==4.66.1" - }, - "triton": { - "hashes": [ - "sha256:143582ca31dd89cd982bd3bf53666bab1c7527d41e185f9e3d8a3051ce1b663b", - "sha256:21544e522c02005a626c8ad63d39bdff2f31d41069592919ef281e964ed26446", - "sha256:39f6fb6bdccb3e98f3152e3fbea724f1aeae7d749412bbb1fa9c441d474eba26", - "sha256:66439923a30d5d48399b08a9eae10370f6c261a5ec864a64983bae63152d39d7", - "sha256:81a96d110a738ff63339fc892ded095b31bd0d205e3aace262af8400d40b6fa8", - "sha256:82fc5aeeedf6e36be4e4530cbdcba81a09d65c18e02f52dc298696d45721f3bd", - "sha256:919b06453f0033ea52c13eaf7833de0e57db3178d23d4e04f9fc71c4f2c32bf8", - "sha256:ae4bb8a91de790e1866405211c4d618379781188f40d5c4c399766914e84cd94" - ], - "version": "==2.1.0" - }, - "typing-extensions": { - "hashes": [ - "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", - "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef" - ], - "markers": "python_version >= '3.8'", - "version": "==4.8.0" - }, - "urllib3": { - "hashes": [ - "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3", - "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54" - ], - "markers": "python_version >= '3.8'", - "version": "==2.1.0" - }, - "wandb": { - "hashes": [ - "sha256:150842447d355d90dc7f368b824951a625e5b2d1be355a00e99b11b73728bc1f", - "sha256:1975ff88c5024923c3321c93cfefb8d9b871543c0b009f34001bf0f31e444b04" - ], - "markers": "python_version >= '3.6'", - "version": "==0.12.21" - }, - "webdataset": { - "hashes": [ - "sha256:7eefdcdf46acc01e80cb6c212acb267e046011c75d4ebdf6a2769f5312eed619", - "sha256:a8ef8809e8c6b32423e40811a529b76a0799d93e3eba1bd1671640906a5fe138" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==0.2.79" - }, - "yarl": { - "hashes": [ - "sha256:09c19e5f4404574fcfb736efecf75844ffe8610606f3fccc35a1515b8b6712c4", - "sha256:0ab5baaea8450f4a3e241ef17e3d129b2143e38a685036b075976b9c415ea3eb", - "sha256:0d155a092bf0ebf4a9f6f3b7a650dc5d9a5bbb585ef83a52ed36ba46f55cc39d", - "sha256:126638ab961633f0940a06e1c9d59919003ef212a15869708dcb7305f91a6732", - "sha256:1a0a4f3aaa18580038cfa52a7183c8ffbbe7d727fe581300817efc1e96d1b0e9", - "sha256:1d93461e2cf76c4796355494f15ffcb50a3c198cc2d601ad8d6a96219a10c363", - "sha256:26a1a8443091c7fbc17b84a0d9f38de34b8423b459fb853e6c8cdfab0eacf613", - "sha256:271d63396460b6607b588555ea27a1a02b717ca2e3f2cf53bdde4013d7790929", - "sha256:28a108cb92ce6cf867690a962372996ca332d8cda0210c5ad487fe996e76b8bb", - "sha256:29beac86f33d6c7ab1d79bd0213aa7aed2d2f555386856bb3056d5fdd9dab279", - "sha256:2c757f64afe53a422e45e3e399e1e3cf82b7a2f244796ce80d8ca53e16a49b9f", - "sha256:2dad8166d41ebd1f76ce107cf6a31e39801aee3844a54a90af23278b072f1ccf", - "sha256:2dc72e891672343b99db6d497024bf8b985537ad6c393359dc5227ef653b2f17", - "sha256:2f3c8822bc8fb4a347a192dd6a28a25d7f0ea3262e826d7d4ef9cc99cd06d07e", - "sha256:32435d134414e01d937cd9d6cc56e8413a8d4741dea36af5840c7750f04d16ab", - "sha256:3cfa4dbe17b2e6fca1414e9c3bcc216f6930cb18ea7646e7d0d52792ac196808", - "sha256:3d5434b34100b504aabae75f0622ebb85defffe7b64ad8f52b8b30ec6ef6e4b9", - "sha256:4003f380dac50328c85e85416aca6985536812c082387255c35292cb4b41707e", - "sha256:44e91a669c43f03964f672c5a234ae0d7a4d49c9b85d1baa93dec28afa28ffbd", - "sha256:4a14907b597ec55740f63e52d7fee0e9ee09d5b9d57a4f399a7423268e457b57", - "sha256:4ce77d289f8d40905c054b63f29851ecbfd026ef4ba5c371a158cfe6f623663e", - "sha256:4d6d74a97e898c1c2df80339aa423234ad9ea2052f66366cef1e80448798c13d", - "sha256:51382c72dd5377861b573bd55dcf680df54cea84147c8648b15ac507fbef984d", - "sha256:525cd69eff44833b01f8ef39aa33a9cc53a99ff7f9d76a6ef6a9fb758f54d0ff", - "sha256:53ec65f7eee8655bebb1f6f1607760d123c3c115a324b443df4f916383482a67", - "sha256:5f74b015c99a5eac5ae589de27a1201418a5d9d460e89ccb3366015c6153e60a", - "sha256:6280353940f7e5e2efaaabd686193e61351e966cc02f401761c4d87f48c89ea4", - "sha256:632c7aeb99df718765adf58eacb9acb9cbc555e075da849c1378ef4d18bf536a", - "sha256:6465d36381af057d0fab4e0f24ef0e80ba61f03fe43e6eeccbe0056e74aadc70", - "sha256:66a6dbf6ca7d2db03cc61cafe1ee6be838ce0fbc97781881a22a58a7c5efef42", - "sha256:6d350388ba1129bc867c6af1cd17da2b197dff0d2801036d2d7d83c2d771a682", - "sha256:7217234b10c64b52cc39a8d82550342ae2e45be34f5bff02b890b8c452eb48d7", - "sha256:721ee3fc292f0d069a04016ef2c3a25595d48c5b8ddc6029be46f6158d129c92", - "sha256:72a57b41a0920b9a220125081c1e191b88a4cdec13bf9d0649e382a822705c65", - "sha256:73cc83f918b69110813a7d95024266072d987b903a623ecae673d1e71579d566", - "sha256:778df71c8d0c8c9f1b378624b26431ca80041660d7be7c3f724b2c7a6e65d0d6", - "sha256:79e1df60f7c2b148722fb6cafebffe1acd95fd8b5fd77795f56247edaf326752", - "sha256:7c86d0d0919952d05df880a1889a4f0aeb6868e98961c090e335671dea5c0361", - "sha256:7eaf13af79950142ab2bbb8362f8d8d935be9aaf8df1df89c86c3231e4ff238a", - "sha256:828235a2a169160ee73a2fcfb8a000709edf09d7511fccf203465c3d5acc59e4", - "sha256:8535e111a064f3bdd94c0ed443105934d6f005adad68dd13ce50a488a0ad1bf3", - "sha256:88d2c3cc4b2f46d1ba73d81c51ec0e486f59cc51165ea4f789677f91a303a9a7", - "sha256:8a2538806be846ea25e90c28786136932ec385c7ff3bc1148e45125984783dc6", - "sha256:8dab30b21bd6fb17c3f4684868c7e6a9e8468078db00f599fb1c14e324b10fca", - "sha256:8f18a7832ff85dfcd77871fe677b169b1bc60c021978c90c3bb14f727596e0ae", - "sha256:946db4511b2d815979d733ac6a961f47e20a29c297be0d55b6d4b77ee4b298f6", - "sha256:96758e56dceb8a70f8a5cff1e452daaeff07d1cc9f11e9b0c951330f0a2396a7", - "sha256:9a172c3d5447b7da1680a1a2d6ecdf6f87a319d21d52729f45ec938a7006d5d8", - "sha256:9a5211de242754b5e612557bca701f39f8b1a9408dff73c6db623f22d20f470e", - "sha256:9df9a0d4c5624790a0dea2e02e3b1b3c69aed14bcb8650e19606d9df3719e87d", - "sha256:aa4643635f26052401750bd54db911b6342eb1a9ac3e74f0f8b58a25d61dfe41", - "sha256:aed37db837ecb5962469fad448aaae0f0ee94ffce2062cf2eb9aed13328b5196", - "sha256:af52725c7c39b0ee655befbbab5b9a1b209e01bb39128dce0db226a10014aacc", - "sha256:b0b8c06afcf2bac5a50b37f64efbde978b7f9dc88842ce9729c020dc71fae4ce", - "sha256:b61e64b06c3640feab73fa4ff9cb64bd8182de52e5dc13038e01cfe674ebc321", - "sha256:b7831566595fe88ba17ea80e4b61c0eb599f84c85acaa14bf04dd90319a45b90", - "sha256:b8bc5b87a65a4e64bc83385c05145ea901b613d0d3a434d434b55511b6ab0067", - "sha256:b8d51817cf4b8d545963ec65ff06c1b92e5765aa98831678d0e2240b6e9fd281", - "sha256:b9f9cafaf031c34d95c1528c16b2fa07b710e6056b3c4e2e34e9317072da5d1a", - "sha256:bb72d2a94481e7dc7a0c522673db288f31849800d6ce2435317376a345728225", - "sha256:c25ec06e4241e162f5d1f57c370f4078797ade95c9208bd0c60f484834f09c96", - "sha256:c405d482c320a88ab53dcbd98d6d6f32ada074f2d965d6e9bf2d823158fa97de", - "sha256:c4472fe53ebf541113e533971bd8c32728debc4c6d8cc177f2bff31d011ec17e", - "sha256:c4b1efb11a8acd13246ffb0bee888dd0e8eb057f8bf30112e3e21e421eb82d4a", - "sha256:c5f3faeb8100a43adf3e7925d556801d14b5816a0ac9e75e22948e787feec642", - "sha256:c6f034386e5550b5dc8ded90b5e2ff7db21f0f5c7de37b6efc5dac046eb19c10", - "sha256:c99ddaddb2fbe04953b84d1651149a0d85214780e4d0ee824e610ab549d98d92", - "sha256:ca6b66f69e30f6e180d52f14d91ac854b8119553b524e0e28d5291a724f0f423", - "sha256:cccdc02e46d2bd7cb5f38f8cc3d9db0d24951abd082b2f242c9e9f59c0ab2af3", - "sha256:cd49a908cb6d387fc26acee8b7d9fcc9bbf8e1aca890c0b2fdfd706057546080", - "sha256:cf7a4e8de7f1092829caef66fd90eaf3710bc5efd322a816d5677b7664893c93", - "sha256:cfd77e8e5cafba3fb584e0f4b935a59216f352b73d4987be3af51f43a862c403", - "sha256:d34c4f80956227f2686ddea5b3585e109c2733e2d4ef12eb1b8b4e84f09a2ab6", - "sha256:d61a0ca95503867d4d627517bcfdc28a8468c3f1b0b06c626f30dd759d3999fd", - "sha256:d81657b23e0edb84b37167e98aefb04ae16cbc5352770057893bd222cdc6e45f", - "sha256:d92d897cb4b4bf915fbeb5e604c7911021a8456f0964f3b8ebbe7f9188b9eabb", - "sha256:dd318e6b75ca80bff0b22b302f83a8ee41c62b8ac662ddb49f67ec97e799885d", - "sha256:dd952b9c64f3b21aedd09b8fe958e4931864dba69926d8a90c90d36ac4e28c9a", - "sha256:e0e7e83f31e23c5d00ff618045ddc5e916f9e613d33c5a5823bc0b0a0feb522f", - "sha256:e0f17d1df951336a02afc8270c03c0c6e60d1f9996fcbd43a4ce6be81de0bd9d", - "sha256:e2a16ef5fa2382af83bef4a18c1b3bcb4284c4732906aa69422cf09df9c59f1f", - "sha256:e36021db54b8a0475805acc1d6c4bca5d9f52c3825ad29ae2d398a9d530ddb88", - "sha256:e73db54c967eb75037c178a54445c5a4e7461b5203b27c45ef656a81787c0c1b", - "sha256:e741bd48e6a417bdfbae02e088f60018286d6c141639359fb8df017a3b69415a", - "sha256:f7271d6bd8838c49ba8ae647fc06469137e1c161a7ef97d778b72904d9b68696", - "sha256:fc391e3941045fd0987c77484b2799adffd08e4b6735c4ee5f054366a2e1551d", - "sha256:fc94441bcf9cb8c59f51f23193316afefbf3ff858460cb47b5758bf66a14d130", - "sha256:fe34befb8c765b8ce562f0200afda3578f8abb159c76de3ab354c80b72244c41", - "sha256:fe8080b4f25dfc44a86bedd14bc4f9d469dfc6456e6f3c5d9077e81a5fedfba7", - "sha256:ff34cb09a332832d1cf38acd0f604c068665192c6107a439a92abfd8acf90fe2" - ], - "markers": "python_version >= '3.7'", - "version": "==1.9.3" - } - }, - "develop": {} -} diff --git a/src/images/Diffusion/README.md b/src/images/Diffusion/README.md deleted file mode 100644 index b17d0b7d7056d0366fea17de7532f7f95b5cc426..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# AI-generated image detection - -This is a group project developed by a team of two individuals. - -## Managing Python packages - -Use of `pipenv` is recommended. The required packages are in `Pipfile`, and can be installed using `pipenv install`. - -## Scraping script for Reddit - -`python scrape.py --subreddit midjourney --flair Showcase` - -This command will scrape the midjourney subreddit, and filter posts that contain the "Showcase" flair. The default number of images to scrape is 30000. The output will contain a parquet file containing metadata, and a csv file containing the urls. - -`img2dataset --url_list=urls/midjourney.csv --output_folder=data/midjourney --thread_count=64 --resize_mode=no --output_format=webdataset` - -This command will download the images in the webdataset format. - - -## Laion script for real images - -`wget -l1 -r --no-parent https://the-eye.eu/public/AI/cah/laion400m-met-release/laion400m-meta/ -mv the-eye.eu/public/AI/cah/laion400m-met-release/laion400m-meta/ .` - -This command will download a 50GB url metadata dataset in 32 parquet files. - -`sample_laion_script.ipynb` - -This script consolidates the parquet files, excludes NSFW images, and selects a subset of 224,917 images. - -`combine_laion_script` - -This script combines the outputs from earlier into 1 parquet file. - -`img2dataset --url_list urls/laion.parquet --input_format "parquet" --url_col "URL" --caption_col "TEXT" --skip_reencode True --output_format webdataset --output_folder data/laion400m_data --processes_count 16 --thread_count 128 --resize_mode no --save_additional_columns '["NSFW","similarity","LICENSE"]' --enable_wandb True` - -This command will download the images in the webdataset format. - - -## Data splitting, preprocessing and loading - -`data_split.py` splits the data according to 80/10/10. The number of samples: - -``` -./data/laion400m_data: (115346, 14418, 14419) -./data/genai-images/StableDiffusion: (22060, 2757, 2758) -./data/genai-images/midjourney: (21096, 2637, 2637) -./data/genai-images/dalle2: (13582, 1697, 1699) -./data/genai-images/dalle3: (12027, 1503, 1504) -``` - -Each sample contains image, target label(1 for GenAI images), and domain label(denoting which generator the image is from). The meaning of the domain label is: - -``` -DOMAIN_LABELS = { - 0: "laion", - 1: "StableDiffusion", - 2: "dalle2", - 3: "dalle3", - 4: "midjourney" -} -``` - -The `load_dataloader()` function in `dataloader.py` returns a `torchdata.dataloader2.DataLoader2` given a list of domains for GenAI images(subset of `[1, 2, 3, 4]`, LAION will always be included). When building the training dataset, data augmentation and class balanced sampling are applied. It is very memory intensive(>20G) and takes some time to fill its buffer before producing batches. Use the dataloader in this way: - -``` -for epoch in range(10): - dl.seed(epoch) - for d in dl: - model(d) -dl.shutdown() -``` diff --git a/src/images/Diffusion/combine_laion_script.ipynb b/src/images/Diffusion/combine_laion_script.ipynb deleted file mode 100644 index 094d7a45d1b83997e79631c9a1490bb3b07bcb11..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/combine_laion_script.ipynb +++ /dev/null @@ -1,117 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pip install pyspark" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "current_directory = os.getcwd()\n", - "print(current_directory)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "os.chdir(current_directory)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from pyspark.sql import SparkSession\n", - "from pyspark.sql.functions import col\n", - "\n", - "spark = SparkSession.builder.appName(\"CombineParquetFiles\").config(\"spark.executor.memory\", \"8g\").config(\"spark.executor.cores\", \"4\").config(\"spark.executor.instances\", \"3\").config(\"spark.dynamicAllocation.enabled\", \"true\").config(\"spark.task.maxFailures\", 10).getOrCreate()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "parquet_directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_sampled'\n", - "\n", - "output_parquet_file = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_combined'\n", - "\n", - "df = spark.read.parquet(parquet_directory_path)\n", - "\n", - "df_coalesced = df.coalesce(1)\n", - "\n", - "df_coalesced.write.mode('overwrite').parquet(output_parquet_file)\n", - "\n", - "row_count = df.count()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(row_count)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "parquet_directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_combined/part-00000-0190eea7-02ac-4ea0-86fd-0722308c0c58-c000.snappy.parquet'\n", - "\n", - "df = spark.read.parquet(parquet_directory_path)\n", - "\n", - "df.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(df.count())" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "bloom", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/images/Diffusion/data_split.py b/src/images/Diffusion/data_split.py deleted file mode 100644 index 01c10ad0d88d2ce5ae46487072c5e06d7ef23389..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/data_split.py +++ /dev/null @@ -1,80 +0,0 @@ -import glob -import json - -import webdataset as wds - - -def split_dataset(path, n_train, n_val, n_test, label, domain_label): - max_file_size = 1000 - input_files = glob.glob(path + "/*.tar") - src = wds.WebDataset(input_files) - - train_path_prefix = path + "/train" - val_path_prefix = path + "/val" - test_path_prefix = path + "/test" - - def write_split(dataset, prefix, start, end): - n_split = end - start - output_files = [ - f"{prefix}_{i}.tar" for i in range(n_split // max_file_size + 1) - ] - for i, output_file in enumerate(output_files): - print(f"Writing {output_file}") - with wds.TarWriter(output_file) as dst: - for sample in dataset.slice( - start + i * max_file_size, - min(start + (i + 1) * max_file_size, end), - ): - new_sample = { - "__key__": sample["__key__"], - "jpg": sample["jpg"], - "label.cls": label, - "domain_label.cls": domain_label, - } - dst.write(new_sample) - - write_split(src, train_path_prefix, 0, n_train) - write_split(src, val_path_prefix, n_train, n_train + n_val) - write_split( - src, - test_path_prefix, - n_train + n_val, - n_train + n_val + n_test, - ) - - -def calculate_sizes(path): - stat_files = glob.glob(path + "/*_stats.json") - total = 0 - for f in stat_files: - with open(f) as stats: - total += json.load(stats)["successes"] - n_train = int(total * 0.8) - n_val = int(total * 0.1) - n_test = total - n_train - n_val - - return n_train, n_val, n_test - - -if __name__ == "__main__": - - paths = [ - "./data/laion400m_data", - "./data/genai-images/StableDiffusion", - "./data/genai-images/midjourney", - "./data/genai-images/dalle2", - "./data/genai-images/dalle3", - ] - - sizes = [] - for p in paths: - res = calculate_sizes(p) - sizes.append(res) - - domain_labels = [0, 1, 4, 2, 3] - - for i, p in enumerate(paths): - print(f"{p}: {sizes[i]}") - label = 0 if i == 0 else 1 - print(label, domain_labels[i]) - split_dataset(p, *calculate_sizes(p), label, domain_labels[i]) diff --git a/src/images/Diffusion/dataloader.py b/src/images/Diffusion/dataloader.py deleted file mode 100644 index 300051bf19140082b7db276492eadb13cb70e1d9..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/dataloader.py +++ /dev/null @@ -1,228 +0,0 @@ -import argparse -import collections -import random -from typing import Iterator - -import cv2 -import numpy as np -import torchdata.datapipes as dp -from imwatermark import WatermarkEncoder -from PIL import ( - Image, - ImageFile, -) -from torch.utils.data import DataLoader -from torchdata.datapipes.iter import ( - Concater, - FileLister, - FileOpener, - SampleMultiplexer, -) -from torchvision.transforms import v2 -from tqdm import tqdm - -ImageFile.LOAD_TRUNCATED_IMAGES = True -Image.MAX_IMAGE_PIXELS = 1000000000 - -encoder = WatermarkEncoder() -encoder.set_watermark("bytes", b"test") - - -DOMAIN_LABELS = { - 0: "laion", - 1: "StableDiffusion", - 2: "dalle2", - 3: "dalle3", - 4: "midjourney", -} - -N_SAMPLES = { - 0: (115346, 14418, 14419), - 1: (22060, 2757, 2758), - 4: (21096, 2637, 2637), - 2: (13582, 1697, 1699), - 3: (12027, 1503, 1504), -} - - -@dp.functional_datapipe("collect_from_workers") -class WorkerResultCollector(dp.iter.IterDataPipe): - def __init__(self, source: dp.iter.IterDataPipe): - self.source = source - - def __iter__(self) -> Iterator: - yield from self.source - - def is_replicable(self) -> bool: - """Method to force data back to main process""" - return False - - -def crop_bottom(image, cutoff=16): - return image[:, :-cutoff, :] - - -def random_gaussian_blur(image, p=0.01): - if random.random() < p: - return v2.functional.gaussian_blur(image, kernel_size=5) - return image - - -def random_invisible_watermark(image, p=0.2): - image_np = np.array(image) - image_np = np.transpose(image_np, (1, 2, 0)) - - if image_np.ndim == 2: # Grayscale image - image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR) - elif image_np.shape[2] == 4: # RGBA image - image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR) - - # print(image_np.shape) - if image_np.shape[0] < 256 or image_np.shape[1] < 256: - image_np = cv2.resize( - image_np, - (256, 256), - interpolation=cv2.INTER_AREA, - ) - if random.random() < p: - return encoder.encode(image_np, method="dwtDct") - return image_np - - -def build_transform(split: str): - train_transform = v2.Compose( - [ - v2.Lambda(crop_bottom), - v2.RandomCrop((256, 256), pad_if_needed=True), - v2.Lambda(random_gaussian_blur), - v2.RandomGrayscale(p=0.05), - v2.Lambda(random_invisible_watermark), - v2.ToImage(), - ], - ) - - eval_transform = v2.Compose( - [ - v2.CenterCrop((256, 256)), - ], - ) - transform = train_transform if split == "train" else eval_transform - - return transform - - -def dp_to_tuple_train(input_dict): - transform = build_transform("train") - return ( - transform(input_dict[".jpg"]), - input_dict[".label.cls"], - input_dict[".domain_label.cls"], - ) - - -def dp_to_tuple_eval(input_dict): - transform = build_transform("eval") - return ( - transform(input_dict[".jpg"]), - input_dict[".label.cls"], - input_dict[".domain_label.cls"], - ) - - -def load_dataset(domains: list[int], split: str): - - laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar") - genai_lister = { - d: FileLister( - f"./data/genai-images/{DOMAIN_LABELS[d]}", - f"{split}*.tar", - ) - for d in domains - if DOMAIN_LABELS[d] != "laion" - } - weight_genai = 1 / len(genai_lister) - - def open_lister(lister): - opener = FileOpener(lister, mode="b") - return opener.load_from_tar().routed_decode().webdataset() - - buffer_size1 = 100 if split == "train" else 10 - buffer_size2 = 100 if split == "train" else 10 - - if split != "train": - all_lister = [laion_lister] + list(genai_lister.values()) - dp = open_lister(Concater(*all_lister)).sharding_filter() - else: - laion_dp = ( - open_lister(laion_lister.shuffle()) - .cycle() - .sharding_filter() - .shuffle(buffer_size=buffer_size1) - ) - genai_dp = { - open_lister(genai_lister[d].shuffle()) - .cycle() - .sharding_filter() - .shuffle(buffer_size=buffer_size1): weight_genai - for d in domains - if DOMAIN_LABELS[d] != "laion" - } - dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle( - buffer_size=buffer_size2, - ) - - if split == "train": - dp = dp.map(dp_to_tuple_train) - else: - dp = dp.map(dp_to_tuple_eval) - - return dp - - -def load_dataloader( - domains: list[int], - split: str, - batch_size: int = 32, - num_workers: int = 4, -): - dp = load_dataset(domains, split) - # if split == "train": - # dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42) - dp = dp.batch(batch_size).collate() - dl = DataLoader( - dp, - batch_size=None, - num_workers=num_workers, - pin_memory=True, - ) - - return dl - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - args = parser.parse_args() - - # testing code - dl = load_dataloader([0, 1], "train", num_workers=8) - y_dist = collections.Counter() - d_dist = collections.Counter() - - for i, (img, y, d) in tqdm(enumerate(dl)): - if i % 100 == 0: - print(y, d) - if i == 400: - break - y_dist.update(y.numpy()) - d_dist.update(d.numpy()) - - print("class label") - for label in sorted(y_dist): - frequency = y_dist[label] / sum(y_dist.values()) - print(f"• {label}: {frequency:.2%} ({y_dist[label]})") - - print("domain label") - for label in sorted(d_dist): - frequency = d_dist[label] / sum(d_dist.values()) - print(f"• {label}: {frequency:.2%} ({d_dist[label]})") diff --git a/src/images/Diffusion/diffusion_data_loader.py b/src/images/Diffusion/diffusion_data_loader.py deleted file mode 100644 index a0dab3f192036d3a9698bc03b6ad340b93253f81..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/diffusion_data_loader.py +++ /dev/null @@ -1,233 +0,0 @@ -import argparse -import collections -import glob -import os -import random -from typing import Iterator - -import cv2 -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchdata as td -import torchdata.datapipes as dp -from imwatermark import WatermarkEncoder -from PIL import ( - Image, - ImageFile, -) -from torch.utils.data import ( - DataLoader, - RandomSampler, -) -from torchdata.dataloader2 import ( - DataLoader2, - MultiProcessingReadingService, -) -from torchdata.datapipes.iter import ( - Concater, - FileLister, - FileOpener, - SampleMultiplexer, -) -from torchvision.transforms import v2 -from tqdm import tqdm -from utils_sampling import UnderSamplerIterDataPipe - -ImageFile.LOAD_TRUNCATED_IMAGES = True -Image.MAX_IMAGE_PIXELS = 1000000000 - -encoder = WatermarkEncoder() -encoder.set_watermark("bytes", b"test") - -DOMAIN_LABELS = { - 0: "laion", - 1: "StableDiffusion", - 2: "dalle2", - 3: "dalle3", - 4: "midjourney", -} - -N_SAMPLES = { - 0: (115346, 14418, 14419), - 1: (22060, 2757, 2758), - 4: (21096, 2637, 2637), - 2: (13582, 1697, 1699), - 3: (12027, 1503, 1504), -} - - -@dp.functional_datapipe("collect_from_workers") -class WorkerResultCollector(dp.iter.IterDataPipe): - def __init__(self, source: dp.iter.IterDataPipe): - self.source = source - - def __iter__(self) -> Iterator: - yield from self.source - - def is_replicable(self) -> bool: - """Method to force data back to main process""" - return False - - -def crop_bottom(image, cutoff=16): - return image[:, :-cutoff, :] - - -def random_gaussian_blur(image, p=0.01): - if random.random() < p: - return v2.functional.gaussian_blur(image, kernel_size=5) - return image - - -def random_invisible_watermark(image, p=0.2): - image_np = np.array(image) - image_np = np.transpose(image_np, (1, 2, 0)) - - if image_np.ndim == 2: # Grayscale image - image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR) - elif image_np.shape[2] == 4: # RGBA image - image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR) - - # print(image_np.shape) - if image_np.shape[0] < 256 or image_np.shape[1] < 256: - image_np = cv2.resize( - image_np, (256, 256), interpolation=cv2.INTER_AREA - ) - if random.random() < p: - return encoder.encode(image_np, method="dwtDct") - return image_np - - -def build_transform(split: str): - train_transform = v2.Compose( - [ - v2.Lambda(crop_bottom), - v2.RandomCrop((256, 256), pad_if_needed=True), - v2.Lambda(random_gaussian_blur), - v2.RandomGrayscale(p=0.05), - v2.Lambda(random_invisible_watermark), - v2.ToImage(), - ] - ) - - eval_transform = v2.Compose( - [ - v2.CenterCrop((256, 256)), - ] - ) - transform = train_transform if split == "train" else eval_transform - - return transform - - -def dp_to_tuple_train(input_dict): - transform = build_transform("train") - return ( - transform(input_dict[".jpg"]), - input_dict[".label.cls"], - input_dict[".domain_label.cls"], - ) - - -def dp_to_tuple_eval(input_dict): - transform = build_transform("eval") - return ( - transform(input_dict[".jpg"]), - input_dict[".label.cls"], - input_dict[".domain_label.cls"], - ) - - -def load_dataset(domains: list[int], split: str): - laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar") - genai_lister = { - d: FileLister( - f"./data/genai-images/{DOMAIN_LABELS[d]}", f"{split}*.tar" - ) - for d in domains - if DOMAIN_LABELS[d] != "laion" - } - weight_genai = 1 / len(genai_lister) - - def open_lister(lister): - opener = FileOpener(lister, mode="b") - return opener.load_from_tar().routed_decode().webdataset() - - buffer_size1 = 100 if split == "train" else 10 - buffer_size2 = 100 if split == "train" else 10 - - if split != "train": - all_lister = [laion_lister] + list(genai_lister.values()) - dp = open_lister(Concater(*all_lister)).sharding_filter() - else: - laion_dp = ( - open_lister(laion_lister.shuffle()) - .cycle() - .sharding_filter() - .shuffle(buffer_size=buffer_size1) - ) - genai_dp = { - open_lister(genai_lister[d].shuffle()) - .cycle() - .sharding_filter() - .shuffle( - buffer_size=buffer_size1, - ): weight_genai - for d in domains - if DOMAIN_LABELS[d] != "laion" - } - dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle( - buffer_size=buffer_size2 - ) - - if split == "train": - dp = dp.map(dp_to_tuple_train) - else: - dp = dp.map(dp_to_tuple_eval) - - return dp - - -def load_dataloader( - domains: list[int], split: str, batch_size: int = 32, num_workers: int = 4 -): - dp = load_dataset(domains, split) - # if split == "train": - # dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42) - dp = dp.batch(batch_size).collate() - dl = DataLoader( - dp, batch_size=None, num_workers=num_workers, pin_memory=True - ) - - return dl - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - args = parser.parse_args() - - # testing code - dl = load_dataloader([0, 1], "train", num_workers=8) - y_dist = collections.Counter() - d_dist = collections.Counter() - - for i, (img, y, d) in tqdm(enumerate(dl)): - if i % 100 == 0: - print(y, d) - if i == 400: - break - y_dist.update(y.numpy()) - d_dist.update(d.numpy()) - - print("class label") - for label in sorted(y_dist): - frequency = y_dist[label] / sum(y_dist.values()) - print(f"• {label}: {frequency:.2%} ({y_dist[label]})") - - print("domain label") - for label in sorted(d_dist): - frequency = d_dist[label] / sum(d_dist.values()) - print(f"• {label}: {frequency:.2%} ({d_dist[label]})") diff --git a/src/images/Diffusion/diffusion_model_classifier.py b/src/images/Diffusion/diffusion_model_classifier.py deleted file mode 100644 index 62495f21bccadc860de5ecf5f6a2d7c85bdd2b2a..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/diffusion_model_classifier.py +++ /dev/null @@ -1,242 +0,0 @@ -import argparse -import logging -import os - -import pandas as pd -import pytorch_lightning as pl -import timm -import torch -import torchvision.transforms as transforms -from data_split import * -from dataloader import * -from PIL import Image -from pytorch_lightning.callbacks import ( - EarlyStopping, - ModelCheckpoint, -) -from sklearn.metrics import roc_auc_score -from torchmetrics import ( - Accuracy, - Recall, -) -from utils_sampling import * - -logging.basicConfig( - filename="training.log", filemode="w", level=logging.INFO, force=True -) - - -class ImageClassifier(pl.LightningModule): - def __init__(self, lmd=0): - super().__init__() - self.model = timm.create_model( - "resnet50", pretrained=True, num_classes=1 - ) - self.accuracy = Accuracy(task="binary", threshold=0.5) - self.recall = Recall(task="binary", threshold=0.5) - self.validation_outputs = [] - self.lmd = lmd - - def forward(self, x): - return self.model(x) - - def training_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - print(f"Shape of outputs (training): {outputs.shape}") - print(f"Shape of labels (training): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - logging.info(f"Training Step - ERM loss: {loss.item()}") - loss += self.lmd * (outputs**2).mean() # SD loss penalty - logging.info(f"Training Step - SD loss: {loss.item()}") - return loss - - def validation_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - if outputs.shape == torch.Size([]): - return - - print(f"Shape of outputs (validation): {outputs.shape}") - print(f"Shape of labels (validation): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - preds = torch.sigmoid(outputs) - self.log("val_loss", loss, prog_bar=True, sync_dist=True) - self.log( - "val_acc", - self.accuracy(preds, labels.int()), - prog_bar=True, - sync_dist=True, - ) - self.log( - "val_recall", - self.recall(preds, labels.int()), - prog_bar=True, - sync_dist=True, - ) - output = {"val_loss": loss, "preds": preds, "labels": labels} - self.validation_outputs.append(output) - logging.info(f"Validation Step - Batch loss: {loss.item()}") - return output - - def predict_step(self, batch): - images, label, domain = batch - outputs = self.forward(images).squeeze() - preds = torch.sigmoid(outputs) - return preds, label, domain - - def on_validation_epoch_end(self): - if not self.validation_outputs: - logging.warning("No outputs in validation step to process") - return - preds = torch.cat([x["preds"] for x in self.validation_outputs]) - labels = torch.cat([x["labels"] for x in self.validation_outputs]) - if labels.unique().size(0) == 1: - logging.warning("Only one class in validation step") - return - auc_score = roc_auc_score(labels.cpu(), preds.cpu()) - self.log("val_auc", auc_score, prog_bar=True, sync_dist=True) - logging.info(f"Validation Epoch End - AUC score: {auc_score}") - self.validation_outputs = [] - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005) - return optimizer - - -checkpoint_callback = ModelCheckpoint( - monitor="val_loss", - dirpath="./model_checkpoints/", - filename="image-classifier-{step}-{val_loss:.2f}", - save_top_k=3, - mode="min", - every_n_train_steps=1001, - enable_version_counter=True, -) - -early_stop_callback = EarlyStopping( - monitor="val_loss", - patience=4, - mode="min", -) - - -def load_image(image_path, transform=None): - image = Image.open(image_path).convert("RGB") - - if transform: - image = transform(image) - - return image - - -def predict_single_image(image_path, model, transform=None): - image = load_image(image_path, transform) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - model.to(device) - - image = image.to(device) - - model.eval() - - with torch.no_grad(): - image = image.unsqueeze(0) - output = model(image).squeeze() - print(output) - prediction = torch.sigmoid(output).item() - - return prediction - - -parser = argparse.ArgumentParser() -parser.add_argument( - "--ckpt_path", help="checkpoint to continue from", required=False -) -parser.add_argument( - "--predict", help="predict on test set", action="store_true" -) -parser.add_argument("--reset", help="reset training", action="store_true") -parser.add_argument( - "--predict_image", - help="predict the class of a single image", - action="store_true", -) -parser.add_argument( - "--image_path", - help="path to the image to predict", - type=str, - required=False, -) -args = parser.parse_args() - -train_domains = [0, 1, 4] -val_domains = [0, 1, 4] -lmd_value = 0 - -if args.predict: - test_dl = load_dataloader( - [0, 1, 2, 3, 4], "test", batch_size=128, num_workers=1 - ) - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - trainer = pl.Trainer() - predictions = trainer.predict(model, dataloaders=test_dl) - preds, labels, domains = zip(*predictions) - preds = torch.cat(preds).cpu().numpy() - labels = torch.cat(labels).cpu().numpy() - domains = torch.cat(domains).cpu().numpy() - print(preds.shape, labels.shape, domains.shape) - df = pd.DataFrame({"preds": preds, "labels": labels, "domains": domains}) - filename = "preds-" + args.ckpt_path.split("/")[-1] - df.to_csv(f"outputs/{filename}.csv", index=False) -elif args.predict_image: - image_path = args.image_path - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - - # Define the transformations for the image - transform = transforms.Compose( - [ - transforms.Resize((224, 224)), # Image size expected by ResNet50 - transforms.ToTensor(), - transforms.Normalize( - mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] - ), - ] - ) - - prediction = predict_single_image(image_path, model, transform) - print("prediction", prediction) - - # Output the prediction - print( - f"Prediction for {image_path}: {'Human' if prediction <= 0.001 else 'Generated'}" - ) -else: - train_dl = load_dataloader( - train_domains, "train", batch_size=128, num_workers=4 - ) - logging.info("Training dataloader loaded") - val_dl = load_dataloader(val_domains, "val", batch_size=128, num_workers=4) - logging.info("Validation dataloader loaded") - - if args.reset: - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - else: - model = ImageClassifier(lmd=lmd_value) - trainer = pl.Trainer( - callbacks=[checkpoint_callback, early_stop_callback], - max_steps=20000, - val_check_interval=1000, - check_val_every_n_epoch=None, - ) - trainer.fit( - model=model, - train_dataloaders=train_dl, - val_dataloaders=val_dl, - ckpt_path=args.ckpt_path if not args.reset else None, - ) diff --git a/src/images/Diffusion/evaluation.ipynb b/src/images/Diffusion/evaluation.ipynb deleted file mode 100644 index 91d7641e5b7d085537459e415ed320baada43a25..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/evaluation.ipynb +++ /dev/null @@ -1,187 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import polars as pl\n", - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score, RocCurveDisplay\n", - "\n", - "sns.set()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def pfbeta(labels, predictions, beta=1):\n", - " y_true_count = 0\n", - " ctp = 0\n", - " cfp = 0\n", - "\n", - " for idx in range(len(labels)):\n", - " prediction = min(max(predictions[idx], 0), 1)\n", - " if (labels[idx]):\n", - " y_true_count += 1\n", - " ctp += prediction\n", - " else:\n", - " cfp += prediction\n", - "\n", - " beta_squared = beta * beta\n", - " c_precision = ctp / (ctp + cfp)\n", - " c_recall = ctp / y_true_count\n", - " if (c_precision > 0 and c_recall > 0):\n", - " result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n", - " return result\n", - " else:\n", - " return 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_part_metrics(df: pl.DataFrame, threshold=0.3) -> dict:\n", - " df = df.with_columns((df[\"preds\"] > threshold).alias(\"preds_bin\"))\n", - " metrics = {}\n", - " # binary metrics using the threshold\n", - " metrics[\"accuracy\"] = accuracy_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " metrics[\"precision\"] = precision_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " metrics[\"recall\"] = recall_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " metrics[\"f1\"] = f1_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " # probabilistic F1 (doesn't depend on the threshold)\n", - " metrics[\"pf1\"] = pfbeta(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n", - " # ROC AUC\n", - " metrics[\"roc_auc\"] = roc_auc_score(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n", - " return metrics\n", - "\n", - "\n", - "def get_all_metrics(df: pl.DataFrame, threshold=0.3) -> pd.DataFrame:\n", - " groups = [list(range(5)), [0, 1], [0, 4], [0, 2], [0, 3]]\n", - " group_names = [\"all\", \"StableDiffusion\", \"Midjourney\", \"Dalle2\", \"Dalle3\"]\n", - " all_metrics = []\n", - " for i, g in enumerate(groups):\n", - " subset = df.filter(pl.col(\"domains\").is_in(g))\n", - " metrics = get_part_metrics(subset, threshold=threshold)\n", - " metrics[\"group\"] = group_names[i]\n", - " all_metrics.append(metrics)\n", - " \n", - " return pd.DataFrame(all_metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df1 = pl.read_csv(\"outputs/preds-image-classifier-1.csv\")\n", - "metrics_df1 = get_all_metrics(df1, threshold=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics_df1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df14 = pl.read_csv(\"outputs/preds-image-classifier-14.csv\")\n", - "metrics_df14 = get_all_metrics(df14, threshold=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics_df14" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df142 = pl.read_csv(\"outputs/preds-image-classifier-142.csv\")\n", - "metrics_df142 = get_all_metrics(df142, threshold=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics_df142" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df1423 = pl.read_csv(\"outputs/preds-image-classifier-1423.csv\")\n", - "metrics_df1423 = get_all_metrics(df1423, threshold=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics_df1423" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "GenAI-image-detection-Z_9oJJe7", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/images/Diffusion/model.py b/src/images/Diffusion/model.py deleted file mode 100644 index 5112cf8cfedeb654894128d5bd65babcc3ea73b0..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/model.py +++ /dev/null @@ -1,307 +0,0 @@ -import argparse -import logging -import os - -import pandas as pd -import pytorch_lightning as pl -import timm -import torch -import torch.nn.functional as F -import torchvision.transforms as transforms -from dataloader import load_dataloader -from PIL import Image -from pytorch_lightning.callbacks import ( - EarlyStopping, - ModelCheckpoint, -) -from sklearn.metrics import roc_auc_score -from torchmetrics import ( - Accuracy, - Recall, -) - -logging.basicConfig( - filename="training.log", - filemode="w", - level=logging.INFO, - force=True, -) - - -class ImageClassifier(pl.LightningModule): - def __init__(self, lmd=0): - super().__init__() - self.model = timm.create_model( - "resnet50", - pretrained=True, - num_classes=1, - ) - self.accuracy = Accuracy(task="binary", threshold=0.5) - self.recall = Recall(task="binary", threshold=0.5) - self.validation_outputs = [] - self.lmd = lmd - - def forward(self, x): - return self.model(x) - - def training_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - print(f"Shape of outputs (training): {outputs.shape}") - print(f"Shape of labels (training): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - logging.info(f"Training Step - ERM loss: {loss.item()}") - loss += self.lmd * (outputs**2).mean() # SD loss penalty - logging.info(f"Training Step - SD loss: {loss.item()}") - return loss - - def validation_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - if outputs.shape == torch.Size([]): - return - - print(f"Shape of outputs (validation): {outputs.shape}") - print(f"Shape of labels (validation): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - preds = torch.sigmoid(outputs) - self.log("val_loss", loss, prog_bar=True, sync_dist=True) - self.log( - "val_acc", - self.accuracy(preds, labels.int()), - prog_bar=True, - sync_dist=True, - ) - self.log( - "val_recall", - self.recall(preds, labels.int()), - prog_bar=True, - sync_dist=True, - ) - output = {"val_loss": loss, "preds": preds, "labels": labels} - self.validation_outputs.append(output) - logging.info(f"Validation Step - Batch loss: {loss.item()}") - return output - - def predict_step(self, batch): - images, label, domain = batch - outputs = self.forward(images).squeeze() - preds = torch.sigmoid(outputs) - return preds, label, domain - - def on_validation_epoch_end(self): - if not self.validation_outputs: - logging.warning("No outputs in validation step to process") - return - preds = torch.cat([x["preds"] for x in self.validation_outputs]) - labels = torch.cat([x["labels"] for x in self.validation_outputs]) - if labels.unique().size(0) == 1: - logging.warning("Only one class in validation step") - return - auc_score = roc_auc_score(labels.cpu(), preds.cpu()) - self.log("val_auc", auc_score, prog_bar=True, sync_dist=True) - logging.info(f"Validation Epoch End - AUC score: {auc_score}") - self.validation_outputs = [] - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005) - return optimizer - - -checkpoint_callback = ModelCheckpoint( - monitor="val_loss", - dirpath="./model_checkpoints/", - filename="image-classifier-{step}-{val_loss:.2f}", - save_top_k=3, - mode="min", - every_n_train_steps=1001, - enable_version_counter=True, -) - -early_stop_callback = EarlyStopping( - monitor="val_loss", - patience=4, - mode="min", -) - - -def load_image(image_path, transform=None): - image = Image.open(image_path).convert("RGB") - - if transform: - image = transform(image) - - return image - - -def predict_single_image(image_path, model, transform=None): - image = load_image(image_path, transform) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - model.to(device) - - image = image.to(device) - - model.eval() - - with torch.no_grad(): - image = image.unsqueeze(0) - output = model(image).squeeze() - print(output) - prediction = torch.sigmoid(output).item() - - return prediction - - -parser = argparse.ArgumentParser() -parser.add_argument( - "--ckpt_path", - help="checkpoint to continue from", - required=False, -) -parser.add_argument( - "--predict", - help="predict on test set", - action="store_true", -) -parser.add_argument("--reset", help="reset training", action="store_true") -parser.add_argument( - "--predict_image", - help="predict the class of a single image", - action="store_true", -) -parser.add_argument( - "--image_path", - help="path to the image to predict", - type=str, - required=False, -) -parser.add_argument( - "--dir", - help="path to the images to predict", - type=str, - required=False, -) -parser.add_argument( - "--output_file", - help="path to output file", - type=str, - required=False, -) -args = parser.parse_args() - -train_domains = [0, 1, 4] -val_domains = [0, 1, 4] -lmd_value = 0 - -if args.predict: - test_dl = load_dataloader( - [0, 1, 2, 3, 4], - "test", - batch_size=128, - num_workers=1, - ) - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - trainer = pl.Trainer() - predictions = trainer.predict(model, dataloaders=test_dl) - preds, labels, domains = zip(*predictions) - preds = torch.cat(preds).cpu().numpy() - labels = torch.cat(labels).cpu().numpy() - domains = torch.cat(domains).cpu().numpy() - print(preds.shape, labels.shape, domains.shape) - df = pd.DataFrame({"preds": preds, "labels": labels, "domains": domains}) - filename = "preds-" + args.ckpt_path.split("/")[-1] - df.to_csv(f"outputs/{filename}.csv", index=False) -elif args.predict_image: - image_path = args.image_path - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - - # Define the transformations for the image - # transform = transforms.Compose( - # [ - # transforms.Resize((224, 224)), # Image size expected by ResNet50 - # transforms.ToTensor(), - # transforms.Normalize( - # mean=[0.485, 0.456, 0.406], - # std=[0.229, 0.224, 0.225], - # ), - # ], - # ) - - transform = transforms.Compose( - [ - transforms.CenterCrop((256, 256)), - transforms.ToTensor(), - ], - ) - - prediction = predict_single_image(image_path, model, transform) - print("prediction", prediction) - - # Output the prediction - print( - f"Prediction for {image_path}: " - f"{'Human' if prediction <= 0.001 else 'Generated'}", - ) -elif args.dir is not None: - predictions = [] - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - # Define the transformations for the image - # transform = transforms.Compose( - # [ - # transforms.Resize((224, 224)), # Image size expected by ResNet50 - # transforms.ToTensor(), - # transforms.Normalize( - # mean=[0.485, 0.456, 0.406], - # std=[0.229, 0.224, 0.225], - # ), - # ], - # ) - transform = transforms.Compose( - [ - transforms.CenterCrop((256, 256)), - transforms.ToTensor(), - ], - ) - for root, dirs, files in os.walk(os.path.abspath(args.dir)): - for f_name in files: - f = os.path.join(root, f_name) - print(f"Predicting: {f}") - p = predict_single_image(f, model, transform) - predictions.append([f, f.split("/")[-2], p, p > 0.5]) - print(f"--predicted: {p}") - - df = pd.DataFrame(predictions, columns=["path", "folder", "pred", "class"]) - df.to_csv(args.output_file, index=False) -else: - train_dl = load_dataloader( - train_domains, - "train", - batch_size=128, - num_workers=4, - ) - logging.info("Training dataloader loaded") - val_dl = load_dataloader(val_domains, "val", batch_size=128, num_workers=4) - logging.info("Validation dataloader loaded") - - if args.reset: - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - else: - model = ImageClassifier(lmd=lmd_value) - trainer = pl.Trainer( - callbacks=[checkpoint_callback, early_stop_callback], - max_steps=20000, - val_check_interval=1000, - check_val_every_n_epoch=None, - ) - trainer.fit( - model=model, - train_dataloaders=train_dl, - val_dataloaders=val_dl, - ckpt_path=args.ckpt_path if not args.reset else None, - ) diff --git a/src/images/Diffusion/sample_laion_script.ipynb b/src/images/Diffusion/sample_laion_script.ipynb deleted file mode 100644 index 9d17e1ce1aa0fa189191f30420e2e44a038f4d82..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/sample_laion_script.ipynb +++ /dev/null @@ -1,73 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import dask.dataframe as dd\n", - "from dask.diagnostics import ProgressBar\n", - "import os\n", - "\n", - "directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/'\n", - "\n", - "file_prefix = 'part'\n", - "\n", - "def list_files_with_prefix(directory, prefix):\n", - " file_paths = []\n", - "\n", - " for root, _, files in os.walk(directory):\n", - " for file in files:\n", - " if file.startswith(prefix):\n", - " absolute_path = os.path.join(root, file)\n", - " file_paths.append(absolute_path)\n", - "\n", - " return file_paths\n", - "\n", - "laion_file_paths = list_files_with_prefix(directory_path, file_prefix)\n", - "\n", - "dataframes = [dd.read_parquet(file) for file in laion_file_paths]\n", - "combined_dataframe = dd.multi.concat(dataframes)\n", - "\n", - "with ProgressBar():\n", - " row_count = combined_dataframe.shape[0].compute()\n", - " print(row_count)\n", - "\n", - "filtered_df = combined_dataframe[combined_dataframe['NSFW'] == \"UNLIKELY\"]\n", - "\n", - "num_samples = 225_000\n", - "selected_rows = filtered_df.sample(frac=num_samples / filtered_df.shape[0].compute())\n", - "\n", - "with ProgressBar():\n", - " sampled_rows = selected_rows.compute()\n", - "\n", - "print(len(sampled_rows))\n", - "\n", - "with ProgressBar():\n", - " selected_rows.to_parquet('laion_sampled', write_index=False)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "bloom", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/images/Diffusion/scrape.py b/src/images/Diffusion/scrape.py deleted file mode 100644 index 88f45282c8d6a3b2fb2b35c50c04963ba7e7df62..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/scrape.py +++ /dev/null @@ -1,149 +0,0 @@ -import argparse -import time - -import polars as pl -import requests - - -def call_api(param): - url = "https://api.pullpush.io/reddit/search/submission/" - response = requests.get(url, params=param) - json_data = response.json()["data"] - create_utc = [] - media_id = [] - media_type_ls = [] - post_ids = [] - post_titles = [] - cur_utc = 0 - for submission in json_data: - cur_flair = submission["link_flair_text"] - cur_utc = submission["created_utc"] - media_ls = ( - submission["media_metadata"] - if "media_metadata" in submission.keys() - else None - ) - if param["flair"] is not None and cur_flair != param["flair"]: - continue - if media_ls is None: - continue - for id in media_ls.keys(): - if media_ls[id]["status"] != "valid": - continue - try: - media_type = media_ls[id]["m"] - except: # noqa - # video will error out - continue - if media_type == "image/png": - media_type_ls.append("png") - elif media_type == "image/jpg": - media_type_ls.append("jpg") - else: - continue - create_utc.append(int(cur_utc)) - post_ids.append(submission["id"]) - post_titles.append(submission["title"]) - media_id.append(id) - - df = pl.DataFrame( - { - "create_utc": create_utc, - "media_id": media_id, - "media_type": media_type_ls, - "post_id": post_ids, - "post_title": post_titles, - }, - schema={ - "create_utc": pl.Int64, - "media_id": pl.Utf8, - "media_type": pl.Utf8, - "post_id": pl.Utf8, - "post_title": pl.Utf8, - }, - ) - return df, int(cur_utc) - - -def scraping_loop( - subreddit, - flair, - max_num=30000, - output_name=None, - before=None, -): - collected_all = [] - collected_len = 0 - last_timestamp = int(time.time()) if before is None else before - param = { - "subreddit": subreddit, - "flair": flair, - "before": last_timestamp, - } - while collected_len < max_num: - collected_df, last_timestamp = call_api(param) - if collected_df.shape[0] == 0: - print("No more data, saving current data and exiting...") - break - collected_all.append(collected_df) - collected_len += collected_df.shape[0] - print( - f"collected_len: {collected_len}, " - f"last_timestamp: {last_timestamp}", - ) - param["before"] = last_timestamp - - df = pl.concat(collected_all) - df = ( - df.with_columns( - pl.col("media_id") - .str.replace(r"^", "https://i.redd.it/") - .alias("url1"), - pl.col("create_utc") - .cast(pl.Int64) - .cast(pl.Utf8) - .str.to_datetime("%s") - .alias("time"), - ) - .with_columns( - pl.col("media_type").str.replace(r"^", ".").alias("url2"), - ) - .with_columns( - pl.concat_str( - [pl.col("url1"), pl.col("url2")], - separator="", - ).alias("url"), - ) - .select("time", "url", "post_id", "post_title") - ) - if output_name is None: - output_name = subreddit - df.write_parquet(f"urls/{output_name}.parquet") - df.select("url").write_csv(f"urls/{output_name}.csv", has_header=False) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--subreddit", help="subreddit name") - parser.add_argument("--flair", help="flair filter", default=None, type=str) - parser.add_argument( - "--max_num", - help="max number of posts to scrape", - default=30000, - type=int, - ) - parser.add_argument( - "--output_name", - help="custom output name", - default=None, - ) - parser.add_argument( - "--before", - help="before timestamp", - default=None, - type=int, - ) - - args = parser.parse_args() - - scraping_loop(**args.__dict__) diff --git a/src/images/Diffusion/utils_sampling.py b/src/images/Diffusion/utils_sampling.py deleted file mode 100644 index 0c7162d54d992c35e2ab4b93a2bae67f6ca8716c..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/utils_sampling.py +++ /dev/null @@ -1,94 +0,0 @@ -import collections -import random -from typing import Callable - -from torchdata.datapipes.iter import IterDataPipe - - -def get_second_entry(sample): - return sample[1] - - -class UnderSamplerIterDataPipe(IterDataPipe): - """Dataset wrapper for under-sampling. - - Copied from: https://github.com/MaxHalford/pytorch-resample/blob/master/pytorch_resample/under.py # noqa - Modified to work with multiple labels. - - MIT License - - Copyright (c) 2020 Max Halford - - This method is based on rejection sampling. - - Parameters: - dataset - desired_dist: The desired class distribution. - The keys are the classes whilst the - values are the desired class percentages. - The values are normalised so that sum up - to 1. - label_getter: A function that takes a sample and returns its label. - seed: Random seed for reproducibility. - - Attributes: - actual_dist: The counts of the observed sample labels. - rng: A random number generator instance. - - References: - - https://www.wikiwand.com/en/Rejection_sampling - - """ - - def __init__( - self, - dataset: IterDataPipe, - desired_dist: dict, - label_getter: Callable = get_second_entry, - seed: int = None, - ): - - self.dataset = dataset - self.desired_dist = { - c: p / sum(desired_dist.values()) for c, p in desired_dist.items() - } - self.label_getter = label_getter - self.seed = seed - - self.actual_dist = collections.Counter() - self.rng = random.Random(seed) - self._pivot = None - - def __iter__(self): - - for dp in self.dataset: - y = self.label_getter(dp) - - self.actual_dist[y] += 1 - - # To ease notation - f = self.desired_dist - g = self.actual_dist - - # Check if the pivot needs to be changed - if y != self._pivot: - self._pivot = max(g.keys(), key=lambda y: f[y] / g[y]) - else: - yield dp - continue - - # Determine the sampling ratio if the observed label - # is not the pivot - M = f[self._pivot] / g[self._pivot] - ratio = f[y] / (M * g[y]) - - if ratio < 1 and self.rng.random() < ratio: - yield dp - - @classmethod - def expected_size(cls, n, desired_dist, actual_dist): - M = max( - desired_dist.get(k) / actual_dist.get(k) - for k in set(desired_dist) | set(actual_dist) - ) - return int(n / M) diff --git a/src/images/Diffusion/visualizations.ipynb b/src/images/Diffusion/visualizations.ipynb deleted file mode 100644 index 748068ad1bb8e3bf03a63af15991d0f4fd00537a..0000000000000000000000000000000000000000 --- a/src/images/Diffusion/visualizations.ipynb +++ /dev/null @@ -1,196 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install polars-lts-cpu" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import polars as pl\n", - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def pfbeta(labels, predictions, beta=1):\n", - " y_true_count = 0\n", - " ctp = 0\n", - " cfp = 0\n", - "\n", - " for idx in range(len(labels)):\n", - " prediction = min(max(predictions[idx], 0), 1)\n", - " if (labels[idx]):\n", - " y_true_count += 1\n", - " ctp += prediction\n", - " else:\n", - " cfp += prediction\n", - "\n", - " beta_squared = beta * beta\n", - " c_precision = ctp / (ctp + cfp)\n", - " c_recall = ctp / y_true_count\n", - " if (c_precision > 0 and c_recall > 0):\n", - " result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n", - " return result\n", - " else:\n", - " return 0\n", - "\n", - "def get_part_metrics(df: pl.DataFrame, threshold=0.3) -> dict:\n", - " df = df.with_columns((df[\"preds\"] > threshold).alias(\"preds_bin\"))\n", - " metrics = {}\n", - " # binary metrics using the threshold\n", - " metrics[\"accuracy\"] = accuracy_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " metrics[\"precision\"] = precision_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " metrics[\"recall\"] = recall_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " metrics[\"f1\"] = f1_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n", - " # probabilistic F1 (doesn't depend on the threshold)\n", - " metrics[\"pf1\"] = pfbeta(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n", - " # ROC AUC\n", - " metrics[\"roc_auc\"] = roc_auc_score(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n", - " return metrics\n", - "\n", - "\n", - "def get_all_metrics(df: pl.DataFrame, threshold=0.3) -> pd.DataFrame:\n", - " groups = [list(range(5)), [0, 1], [0, 4], [0, 2], [0, 3]]\n", - " group_names = [\"all\", \"StableDiffusion\", \"Midjourney\", \"Dalle2\", \"Dalle3\"]\n", - " all_metrics = []\n", - " for i, g in enumerate(groups):\n", - " subset = df.filter(pl.col(\"domains\").is_in(g))\n", - " metrics = get_part_metrics(subset, threshold=threshold)\n", - " metrics[\"group\"] = group_names[i]\n", - " all_metrics.append(metrics)\n", - " \n", - " return pd.DataFrame(all_metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load the data from the output files\n", - "df1 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-1.csv')\n", - "df14 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-14.csv')\n", - "df142 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-142.csv')\n", - "df1423 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-1423.csv')\n", - "\n", - "metrics_df1 = get_all_metrics(df1, threshold=0.5)\n", - "metrics_df14 = get_all_metrics(df14, threshold=0.5)\n", - "metrics_df142 = get_all_metrics(df142, threshold=0.5)\n", - "metrics_df1423 = get_all_metrics(df1423, threshold=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics_df1.info()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sns.set()\n", - "\n", - "models = ['StableDiffusion', 'Midjourney', 'Dalle2', 'Dalle3']\n", - "metrics = ['accuracy', 'f1', 'pf1', 'roc_auc']\n", - "\n", - "file_map = {\n", - " ('StableDiffusion',): metrics_df1,\n", - " ('StableDiffusion', 'Midjourney'): metrics_df14,\n", - " ('StableDiffusion', 'Midjourney', 'Dalle2'): metrics_df142,\n", - " ('StableDiffusion', 'Midjourney', 'Dalle2', 'Dalle3'): metrics_df1423,\n", - "}\n", - "\n", - "def create_heatmap_data(metric):\n", - " data = pd.DataFrame(index=models[::-1], columns=models)\n", - " for i, model_x in enumerate(models):\n", - " for j, model_y in enumerate(models[::-1]):\n", - " \n", - " if i == 0:\n", - " relevant_df = metrics_df1\n", - " elif i == 1:\n", - " relevant_df = metrics_df14\n", - " elif i == 2:\n", - " relevant_df = metrics_df142\n", - " else:\n", - " relevant_df = metrics_df1423\n", - "\n", - " # Debugging: print the DataFrame being used and the model_y\n", - " #print(f\"Using DataFrame for {models[:i+1]}, model_y: {model_y}\")\n", - "\n", - " # Extract the metric value\n", - " if model_y in relevant_df['group'].values:\n", - " metric_value = relevant_df[relevant_df['group'] == model_y][metric].values[0]\n", - " # Debugging: print the extracted metric value\n", - " #print(f\"Metric value for {model_y}: {metric_value}\")\n", - " else:\n", - " metric_value = float('nan') # Handle non-existent cases\n", - " # Debugging: print a message for non-existent cases\n", - " #print(f\"No data for combination: {model_x}, {model_y}\")\n", - "\n", - " data.at[model_y, model_x] = metric_value\n", - " \n", - " for col in data.columns:\n", - " data[col] = pd.to_numeric(data[col], errors='coerce')\n", - "\n", - " # Debugging: print the final DataFrame\n", - " # print(f\"Final Data for metric {metric}:\")\n", - " # print(data)\n", - " # print(data.dtypes)\n", - " return data\n", - "\n", - "for metric in metrics:\n", - " plt.figure(figsize=(10, 8))\n", - " sns.heatmap(create_heatmap_data(metric), annot=True, cmap='coolwarm', fmt='.3f')\n", - " plt.title(f\"Heatmap for {metric}\")\n", - " plt.xlabel(\"Trained On (x-axis)\")\n", - " plt.ylabel(\"Tested On (y-axis)\")\n", - " plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "bloom", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/images/README.md b/src/images/README.md deleted file mode 100644 index 5be0ef70860248b08f9fd8d2e42cc90824f161ee..0000000000000000000000000000000000000000 --- a/src/images/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# AI-generated image detection -**(Work In Progress)** - -- [ ] Refactor code -- [ ] Review dependencies -- [ ] Containerize (Docker) -- [ ] Update documentation - -## AI-Generated Image detection - -This part handles the detection of AI-generated images. -The current code contains two classifiers to detect AI-generated images from two types of architectures: -- GANs - -## Model weights - -### 1. CNN Detection - -Run the `download_weights_CNN.sh` script: - -```commandline -bash download_weights_CNN.sh -``` - -Note: you need `wget` installed on your system (it is by default for most Linux systems). - -### 2. Diffusion - -**TODO** - - -## Run the models - -Make sure you have the weights available before doing so. - -**TODO: environments** - -### 1. CNN Detection - -```commandline -python CNN_model_classifier.py -``` -Available options: - -- `-f / --file` (default=`'examples_realfakedir'`) -- `-m / --model_path` (default=`'weights/blur_jpg_prob0.5.pth'`) -- `-c / --crop` (default=`None`): Specify crop size (int) by default, do not crop. -- `--use_cpu`: use cpu (by default uses GPU) -> **TODO: remove (obsolete)** - -Example usage: - -```commandline -python CNN_model_classifier.py -f examples/real.png -m weights/blur_jpg_prob0.5.pth -``` - -### 2. Diffusion detection - -**TODO** - -## References - -Based on: -- https://github.com/hoangthuc701/GenAI-image-detection -- https://github.com/ptmaimai106/DetectGenerateImageByRealImageOnly diff --git a/src/images/Search_Image/Bing_search.py b/src/images/Search_Image/Bing_search.py deleted file mode 100644 index 58ebf8c9fa8ac12ec0213ffd7850e54d85cfb050..0000000000000000000000000000000000000000 --- a/src/images/Search_Image/Bing_search.py +++ /dev/null @@ -1,93 +0,0 @@ -import json -import os -from dotenv import load_dotenv -import requests - -# Load Bing Search API key -load_dotenv() -BING_API_KEY = os.getenv("BING_API_KEY") - -def print_json(obj): - """Print the object as json""" - print(json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': '))) - - -def get_image_urls(search_results): - """ - Extracts image URLs from Bing Visual Search response. - Ref: https://learn.microsoft.com/en-us/bing/search-apis/bing-visual-search/how-to/search-response - - Args: - search_results: A dict containing the Bing VisualSearch response data. - - Returns: - A tuple containing two lists: - - List of image URLs from "PagesIncluding" section. - - List of image URLs from "VisualSearch" section (backup). - """ - - pages_including_urls = [] - visual_search_urls = [] - - if "tags" not in search_results: - return pages_including_urls, visual_search_urls - - # Check for required keys directly - if not any(action.get("actions") for action in search_results["tags"]): - return pages_including_urls, visual_search_urls - - - for action in search_results["tags"]: - for result in action.get("actions", []): - # actions = PagesIncluding, main results - if result["name"] == "PagesIncluding": - pages_including_urls.extend(item["contentUrl"] for item in result["data"]["value"]) - # actions = VisualSearch, back up results - elif result["name"] == "VisualSearch": - visual_search_urls.extend(item["contentUrl"] for item in result["data"]["value"]) - - return pages_including_urls, visual_search_urls - -def reverse_image_search(image_path, subscription_key=BING_API_KEY): - """Performs a reverse image search using the Bing Visual Search API. - - Args: - image_path: The path to the image file to search for. - - Returns: - A list of image URLs found that are similar to the image in the - specified path. - - Raises: - requests.exceptions.RequestException: If the API request fails. - """ - base_uri = "https://api.bing.microsoft.com/v7.0/images/visualsearch" - headers = {"Ocp-Apim-Subscription-Key": subscription_key} - - try: - files = {"image": ("image", open(image_path, "rb"))} - response = requests.post(base_uri, headers=headers, files=files) - response.raise_for_status() - search_results = response.json() - - return search_results - - except requests.exceptions.RequestException as e: - raise requests.exceptions.RequestException(f"API request failed: {e}") - except OSError as e: - raise OSError(f"Error opening image file: {e}") - -if __name__ == "__main__": - # Example usage: - image_path = "data/test_data/human_news.jpg" - try: - search_results = reverse_image_search(image_path) - image_urls, backup_image_urls = get_image_urls(search_results) - - # Print the results - print("Image URLs from PagesIncluding:") - print(image_urls) - print("\nImage URLs from VisualSearch (backup):") - print(backup_image_urls) - except Exception as e: - print(f"An error occurred: {e}") \ No newline at end of file diff --git a/src/images/Search_Image/image_difference.py b/src/images/Search_Image/image_difference.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/images/Search_Image/image_model_share.py b/src/images/Search_Image/image_model_share.py deleted file mode 100644 index 4503f7d3c2b1bbffa2917dab44d9c9fd249e99fa..0000000000000000000000000000000000000000 --- a/src/images/Search_Image/image_model_share.py +++ /dev/null @@ -1,142 +0,0 @@ -from sklearn.metrics import roc_auc_score -from torchmetrics import Accuracy, Recall -import pytorch_lightning as pl -import timm -import torch -from pytorch_lightning.callbacks import Model, EarlyStopping -import logging -from PIL import Image -import torchvision.transforms as transforms -from torchvision.transforms import v2 - -logging.basicConfig(filename='training.log',filemode='w',level=logging.INFO, force=True) -CHECKPOINT = "models/image_classifier/image-classifier-step=8008-val_loss=0.11.ckpt" - - - -class ImageClassifier(pl.LightningModule): - def __init__(self, lmd=0): - super().__init__() - self.model = timm.create_model('resnet50', pretrained=True, num_classes=1) - self.accuracy = Accuracy(task='binary', threshold=0.5) - self.recall = Recall(task='binary', threshold=0.5) - self.validation_outputs = [] - self.lmd = lmd - - def forward(self, x): - return self.model(x) - - def training_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - print(f"Shape of outputs (training): {outputs.shape}") - print(f"Shape of labels (training): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - logging.info(f"Training Step - ERM loss: {loss.item()}") - loss += self.lmd * (outputs ** 2).mean() # SD loss penalty - logging.info(f"Training Step - SD loss: {loss.item()}") - return loss - - def validation_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - if outputs.shape == torch.Size([]): - return - - print(f"Shape of outputs (validation): {outputs.shape}") - print(f"Shape of labels (validation): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - preds = torch.sigmoid(outputs) - self.log('val_loss', loss, prog_bar=True, sync_dist=True) - self.log('val_acc', self.accuracy(preds, labels.int()), prog_bar=True, sync_dist=True) - self.log('val_recall', self.recall(preds, labels.int()), prog_bar=True, sync_dist=True) - output = {"val_loss": loss, "preds": preds, "labels": labels} - self.validation_outputs.append(output) - logging.info(f"Validation Step - Batch loss: {loss.item()}") - return output - - def predict_step(self, batch): - images, label, domain = batch - outputs = self.forward(images).squeeze() - preds = torch.sigmoid(outputs) - return preds, label, domain - - def on_validation_epoch_end(self): - if not self.validation_outputs: - logging.warning("No outputs in validation step to process") - return - preds = torch.cat([x['preds'] for x in self.validation_outputs]) - labels = torch.cat([x['labels'] for x in self.validation_outputs]) - if labels.unique().size(0) == 1: - logging.warning("Only one class in validation step") - return - auc_score = roc_auc_score(labels.cpu(), preds.cpu()) - self.log('val_auc', auc_score, prog_bar=True, sync_dist=True) - logging.info(f"Validation Epoch End - AUC score: {auc_score}") - self.validation_outputs = [] - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005) - return optimizer - - - -def load_image(image_path, transform=None): - image = Image.open(image_path).convert('RGB') - - if transform: - image = transform(image) - - return image - - -def predict_single_image(image_path, model, transform=None): - image = load_image(image_path, transform) - - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - - model.to(device) - - image = image.to(device) - - model.eval() - - with torch.no_grad(): - image = image.unsqueeze(0) - output = model(image).squeeze() - print(output) - prediction = torch.sigmoid(output).item() - - return prediction - - -def image_generation_detection(image_path): - model = ImageClassifier.load_from_checkpoint(CHECKPOINT) - - transform = v2.Compose([ - transforms.ToTensor(), - v2.CenterCrop((256, 256)), - ]) - - prediction = predict_single_image(image_path, model, transform) - print("prediction",prediction) - - result = "" - if prediction <= 0.2: - result += "Most likely human" - image_prediction_label = "HUMAN" - else: - result += "Most likely machine" - image_prediction_label = "MACHINE" - image_confidence = min(1, 0.5 + abs(prediction - 0.2)) - result += f" with confidence = {round(image_confidence * 100, 2)}%" - image_confidence = round(image_confidence * 100, 2) - return image_prediction_label, image_confidence - - -if __name__ == "__main__": - pass diff --git a/src/images/Search_Image/search.py b/src/images/Search_Image/search.py deleted file mode 100644 index 10c0cb91f91f2fbc987479de92c98f867d14a2e0..0000000000000000000000000000000000000000 --- a/src/images/Search_Image/search.py +++ /dev/null @@ -1,56 +0,0 @@ -from google_img_source_search import ReverseImageSearcher -import requests -from io import BytesIO -from PIL import Image -import imagehash -from google_img_source_search import ReverseImageSearcher - -def get_image_from_url(url): - response = requests.get(url) - return Image.open(BytesIO(response.content)) - -def standardize_image(image): - # Convert to RGB if needed - if image.mode in ('RGBA', 'LA'): - background = Image.new('RGB', image.size, (255, 255, 255)) - background.paste(image, mask=image.split()[-1]) - image = background - elif image.mode != 'RGB': - image = image.convert('RGB') - - # Resize to standard size (e.g. 256x256) - standard_size = (256, 256) - image = image.resize(standard_size) - - return image - -def compare_images(image1, image2): - # Standardize both images first - img1_std = standardize_image(image1) - img2_std = standardize_image(image2) - - hash1 = imagehash.average_hash(img1_std) - hash2 = imagehash.average_hash(img2_std) - return hash1 - hash2 # Returns the Hamming distance between the hashes - -if __name__ == '__main__': - image_url = 'https://i.pinimg.com/originals/c4/50/35/c450352ac6ea8645ead206721673e8fb.png' - - # Get the image from URL - url_image = get_image_from_url(image_url) - - # Search image - rev_img_searcher = ReverseImageSearcher() - res = rev_img_searcher.search(image_url) - - for search_item in res: - print(f'Title: {search_item.page_title}') - # print(f'Site: {search_item.page_url}') - print(f'Img: {search_item.image_url}\n') - - # Compare each search result image with the input image - result_image = get_image_from_url(search_item.image_url) - result_difference = compare_images(result_image, url_image) - print(f"Difference with search result: {result_difference}") - if result_difference == 0: - break \ No newline at end of file diff --git a/src/images/Search_Image/search_2.py b/src/images/Search_Image/search_2.py deleted file mode 100644 index 066d250631ee68548445678b5961759a0218cbfc..0000000000000000000000000000000000000000 --- a/src/images/Search_Image/search_2.py +++ /dev/null @@ -1,150 +0,0 @@ -import time -import logging -import requests -from bs4 import BeautifulSoup -from typing import Dict, Optional -from urllib.parse import quote, urlparse - -logging.basicConfig( - filename='error.log', - level=logging.INFO, - format='%(asctime)s | [%(levelname)s]: %(message)s', - datefmt='%m-%d-%Y / %I:%M:%S %p' -) - -class SearchResults: - def __init__(self, results): - self.results = results - - def __str__(self): - output = "" - for result in self.results: - output += "---\n" - output += f"Title: {result.get('title', 'Title not found')}\n" - output += f"Link: {result.get('link', 'Link not found')}\n" - output += "---\n" - return output - -class GoogleReverseImageSearch: - def __init__(self): - self.base_url = "https://www.google.com/searchbyimage" - self.headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"} - self.retry_count = 3 - self.retry_delay = 1 - - def response(self, query: str, image_url: str, max_results: int = 10, delay: int = 1) -> SearchResults: - self._validate_input(query, image_url) - - encoded_query = quote(query) - encoded_image_url = quote(image_url) - - url = f"{self.base_url}?q={encoded_query}&image_url={encoded_image_url}&sbisrc=cr_1_5_2" - - all_results = [] - start_index = 0 - - while len(all_results) < max_results: - if start_index != 0: - time.sleep(delay) - - paginated_url = f"{url}&start={start_index}" - - response = self._make_request(paginated_url) - if response is None: - break - - search_results, valid_content = self._parse_search_results(response.text) - if not valid_content: - logging.warning("Unexpected HTML structure encountered.") - break - - for result in search_results: - if len(all_results) >= max_results: - break - data = self._extract_result_data(result) - if data and data not in all_results: - all_results.append(data) - - start_index += (len(all_results)-start_index) - - if len(all_results) == 0: - logging.warning(f"No results were found for the given query: [{query}], and/or image URL: [{image_url}].") - return "No results found. Please try again with a different query and/or image URL." - else: - return SearchResults(all_results[:max_results]) - - def _validate_input(self, query: str, image_url: str): - if not query: - raise ValueError("Query not found. Please enter a query and try again.") - if not image_url: - raise ValueError("Image URL not found. Please enter an image URL and try again.") - if not self._validate_image_url(image_url): - raise ValueError("Invalid image URL. Please enter a valid image URL and try again.") - - def _validate_image_url(self, url: str) -> bool: - parsed_url = urlparse(url) - path = parsed_url.path.lower() - valid_extensions = (".jpg", ".jpeg", ".png", ".webp") - return any(path.endswith(ext) for ext in valid_extensions) - - def _make_request(self, url: str): - attempts = 0 - while attempts < self.retry_count: - try: - response = requests.get(url, headers=self.headers) - if response.headers.get('Content-Type', '').startswith('text/html'): - response.raise_for_status() - return response - else: - logging.warning("Non-HTML content received.") - return None - except requests.exceptions.HTTPError as http_err: - logging.error(f"HTTP error occurred: {http_err}") - attempts += 1 - time.sleep(self.retry_delay) - except Exception as err: - logging.error(f"An error occurred: {err}") - return None - return None - - def _parse_search_results(self, html_content: str) -> (Optional[list], bool): - try: - soup = BeautifulSoup(html_content, "html.parser") - return soup.find_all('div', class_='g'), True - except Exception as e: - logging.error(f"Error parsing HTML content: {e}") - return None, False - - def _extract_result_data(self, result) -> Dict: - link = result.find('a', href=True)['href'] if result.find('a', href=True) else None - title = result.find('h3').get_text(strip=True) if result.find('h3') else None - return {"link": link, "title": title} if link and title else {} - - -if __name__ == "__main__": - # request = GoogleReverseImageSearch() - - # response = request.response( - # query="Example Query", - # image_url="https://ichef.bbci.co.uk/images/ic/1024xn/p0khzhhl.jpg.webp", - # max_results=5 - # ) - - # print(response) - - # Path to local image - image_path = "data/test_data/towel.jpg" - image_path = "C:\\TTProjects\\prj-nict-ai-content-detection\\data\\test_data\\towel.jpg" - - import json - file_path = image_path - search_url = 'https://yandex.ru/images/search' - files = {'upfile': ('blob', open(file_path, 'rb'), 'image/jpeg')} - params = {'rpt': 'imageview', 'format': 'json', 'request': '{"blocks":[{"block":"b-page_type_search-by-image__link"}]}'} - response = requests.post(search_url, params=params, files=files) - query_string = json.loads(response.content)['blocks'][0]['params']['url'] - img_search_url = search_url + '?' + query_string - print(img_search_url) - - response = requests.get(img_search_url) - print(response.text) \ No newline at end of file diff --git a/src/images/Search_Image/search_yandex.py b/src/images/Search_Image/search_yandex.py deleted file mode 100644 index ee19e528cbda08a991a45ba30a12300b4e8d900a..0000000000000000000000000000000000000000 --- a/src/images/Search_Image/search_yandex.py +++ /dev/null @@ -1,177 +0,0 @@ -import time -import logging -import requests -from bs4 import BeautifulSoup -from typing import Dict, Optional -from urllib.parse import quote, urlparse - -logging.basicConfig( - filename='error.log', - level=logging.INFO, - format='%(asctime)s | [%(levelname)s]: %(message)s', - datefmt='%m-%d-%Y / %I:%M:%S %p' -) - -class SearchResults: - def __init__(self, results): - self.results = results - - def __str__(self): - output = "" - for result in self.results: - output += "---\n" - output += f"Title: {result.get('title', 'Title not found')}\n" - output += f"Link: {result.get('link', 'Link not found')}\n" - output += "---\n" - return output - -class ReverseImageSearch: - def __init__(self): - self.base_url = "https://yandex.ru/images/search" - self.headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"} - self.retry_count = 3 - self.retry_delay = 1 - - def response(self, query: str, image_url: str, max_results: int = 10, delay: int = 1) -> SearchResults: - self._validate_input(query, image_url) - - encoded_query = quote(query) - encoded_image_url = quote(image_url) - - url = f"{self.base_url}?q={encoded_query}&image_url={encoded_image_url}&sbisrc=cr_1_5_2" - - all_results = [] - start_index = 0 - - while len(all_results) < max_results: - if start_index != 0: - time.sleep(delay) - - paginated_url = f"{url}&start={start_index}" - - response = self._make_request(paginated_url) - if response is None: - break - - search_results, valid_content = self._parse_search_results(response.text) - if not valid_content: - logging.warning("Unexpected HTML structure encountered.") - break - - for result in search_results: - if len(all_results) >= max_results: - break - data = self._extract_result_data(result) - if data and data not in all_results: - all_results.append(data) - - start_index += (len(all_results)-start_index) - - if len(all_results) == 0: - logging.warning(f"No results were found for the given query: [{query}], and/or image URL: [{image_url}].") - return "No results found. Please try again with a different query and/or image URL." - else: - return SearchResults(all_results[:max_results]) - - def _validate_input(self, query: str, image_url: str): - if not query: - raise ValueError("Query not found. Please enter a query and try again.") - if not image_url: - raise ValueError("Image URL not found. Please enter an image URL and try again.") - if not self._validate_image_url(image_url): - raise ValueError("Invalid image URL. Please enter a valid image URL and try again.") - - def _validate_image_url(self, url: str) -> bool: - parsed_url = urlparse(url) - path = parsed_url.path.lower() - valid_extensions = (".jpg", ".jpeg", ".png", ".webp") - return any(path.endswith(ext) for ext in valid_extensions) - - def _make_request(self, url: str): - attempts = 0 - while attempts < self.retry_count: - try: - response = requests.get(url, headers=self.headers) - if response.headers.get('Content-Type', '').startswith('text/html'): - response.raise_for_status() - return response - else: - logging.warning("Non-HTML content received.") - return None - except requests.exceptions.HTTPError as http_err: - logging.error(f"HTTP error occurred: {http_err}") - attempts += 1 - time.sleep(self.retry_delay) - except Exception as err: - logging.error(f"An error occurred: {err}") - return None - return None - - def _parse_search_results(self, html_content: str) -> (Optional[list], bool): - try: - soup = BeautifulSoup(html_content, "html.parser") - return soup.find_all('div', class_='g'), True - except Exception as e: - logging.error(f"Error parsing HTML content: {e}") - return None, False - - def _extract_result_data(self, result) -> Dict: - link = result.find('a', href=True)['href'] if result.find('a', href=True) else None - title = result.find('h3').get_text(strip=True) if result.find('h3') else None - return {"link": link, "title": title} if link and title else {} - -def yandex_reverse_image_search(image_url): - # Simulate a user agent to avoid being blocked - headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'} - - try: - response = requests.get(image_url, headers=headers) - response.raise_for_status() # Raise an exception for bad status codes - - # Parse the HTML content - soup = BeautifulSoup(response.content, 'html.parser') - - # Extract image URLs (example - adapt based on Yandex's HTML structure) - image_urls = [img['src'] for img in soup.find_all('img')] - - # Extract related searches (example - adapt based on Yandex's HTML structure) - related_searches = [text for text in soup.find_all(class_="related-searches")] - - return image_urls, related_searches - - except requests.exceptions.RequestException as e: - print(f"Error fetching image: {e}") - return [], [] - - -if __name__ == "__main__": - # request = GoogleReverseImageSearch() - - # response = request.response( - # query="Example Query", - # image_url="https://ichef.bbci.co.uk/images/ic/1024xn/p0khzhhl.jpg.webp", - # max_results=5 - # ) - - # print(response) - - # Path to local image - image_path = "data/test_data/towel.jpg" - image_path = "C:\\TTProjects\\prj-nict-ai-content-detection\\data\\test_data\\towel.jpg" - - import json - file_path = image_path - search_url = 'https://yandex.ru/images/search' - files = {'upfile': ('blob', open(file_path, 'rb'), 'image/jpeg')} - params = {'rpt': 'imageview', 'format': 'json', 'request': '{"blocks":[{"block":"b-page_type_search-by-image__link"}]}'} - response = requests.post(search_url, params=params, files=files) - query_string = json.loads(response.content)['blocks'][0]['params']['url'] - img_search_url = search_url + '?' + query_string - print(img_search_url) - - image_urls, related_searches = yandex_reverse_image_search(img_search_url) - - print("Image URLs:", image_urls) - print("Related Searches:", related_searches) - - \ No newline at end of file diff --git a/src/images/diffusion_data_loader.py b/src/images/diffusion_data_loader.py deleted file mode 100644 index b9e74f785b69e9fb7d73a60887c4551a3464cfe7..0000000000000000000000000000000000000000 --- a/src/images/diffusion_data_loader.py +++ /dev/null @@ -1,229 +0,0 @@ -import argparse -import collections -import random -from typing import Iterator - -import cv2 -import numpy as np -import torchdata.datapipes as dp -from imwatermark import WatermarkEncoder -from PIL import ( - Image, - ImageFile, -) -from torch.utils.data import DataLoader -from torchdata.datapipes.iter import ( - Concater, - FileLister, - FileOpener, - SampleMultiplexer, -) -from torchvision.transforms import v2 -from tqdm import tqdm - -ImageFile.LOAD_TRUNCATED_IMAGES = True -Image.MAX_IMAGE_PIXELS = 1000000000 - -encoder = WatermarkEncoder() -encoder.set_watermark("bytes", b"test") - -DOMAIN_LABELS = { - 0: "laion", - 1: "StableDiffusion", - 2: "dalle2", - 3: "dalle3", - 4: "midjourney", -} - -N_SAMPLES = { - 0: (115346, 14418, 14419), - 1: (22060, 2757, 2758), - 4: (21096, 2637, 2637), - 2: (13582, 1697, 1699), - 3: (12027, 1503, 1504), -} - - -@dp.functional_datapipe("collect_from_workers") -class WorkerResultCollector(dp.iter.IterDataPipe): - def __init__(self, source: dp.iter.IterDataPipe): - self.source = source - - def __iter__(self) -> Iterator: - yield from self.source - - def is_replicable(self) -> bool: - """Method to force data back to main process""" - return False - - -def crop_bottom(image, cutoff=16): - return image[:, :-cutoff, :] - - -def random_gaussian_blur(image, p=0.01): - if random.random() < p: - return v2.functional.gaussian_blur(image, kernel_size=5) - return image - - -def random_invisible_watermark(image, p=0.2): - image_np = np.array(image) - image_np = np.transpose(image_np, (1, 2, 0)) - - if image_np.ndim == 2: # Grayscale image - image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR) - elif image_np.shape[2] == 4: # RGBA image - image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR) - - if image_np.shape[0] < 256 or image_np.shape[1] < 256: - image_np = cv2.resize( - image_np, - (256, 256), - interpolation=cv2.INTER_AREA, - ) - - if random.random() < p: - return encoder.encode(image_np, method="dwtDct") - - return image_np - - -def build_transform(split: str): - train_transform = v2.Compose( - [ - v2.Lambda(crop_bottom), - v2.RandomCrop((256, 256), pad_if_needed=True), - v2.Lambda(random_gaussian_blur), - v2.RandomGrayscale(p=0.05), - v2.Lambda(random_invisible_watermark), - v2.ToImage(), - ], - ) - - eval_transform = v2.Compose( - [ - v2.CenterCrop((256, 256)), - ], - ) - transform = train_transform if split == "train" else eval_transform - - return transform - - -def dp_to_tuple_train(input_dict): - transform = build_transform("train") - return ( - transform(input_dict[".jpg"]), - input_dict[".label.cls"], - input_dict[".domain_label.cls"], - ) - - -def dp_to_tuple_eval(input_dict): - transform = build_transform("eval") - return ( - transform(input_dict[".jpg"]), - input_dict[".label.cls"], - input_dict[".domain_label.cls"], - ) - - -def load_dataset(domains: list[int], split: str): - laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar") - genai_lister = { - d: FileLister( - f"./data/genai-images/{DOMAIN_LABELS[d]}", - f"{split}*.tar", - ) - for d in domains - if DOMAIN_LABELS[d] != "laion" - } - weight_genai = 1 / len(genai_lister) - - def open_lister(lister): - opener = FileOpener(lister, mode="b") - return opener.load_from_tar().routed_decode().webdataset() - - buffer_size1 = 100 if split == "train" else 10 - buffer_size2 = 100 if split == "train" else 10 - - if split != "train": - all_lister = [laion_lister] + list(genai_lister.values()) - dp = open_lister(Concater(*all_lister)).sharding_filter() - else: - laion_dp = ( - open_lister(laion_lister.shuffle()) - .cycle() - .sharding_filter() - .shuffle(buffer_size=buffer_size1) - ) - genai_dp = { - open_lister(genai_lister[d].shuffle()) - .cycle() - .sharding_filter() - .shuffle( - buffer_size=buffer_size1, - ): weight_genai - for d in domains - if DOMAIN_LABELS[d] != "laion" - } - dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle( - buffer_size=buffer_size2, - ) - - if split == "train": - dp = dp.map(dp_to_tuple_train) - else: - dp = dp.map(dp_to_tuple_eval) - - return dp - - -def load_dataloader( - domains: list[int], - split: str, - batch_size: int = 32, - num_workers: int = 4, -): - dp = load_dataset(domains, split) - # if split == "train": - # dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42) - dp = dp.batch(batch_size).collate() - dl = DataLoader( - dp, - batch_size=None, - num_workers=num_workers, - pin_memory=True, - ) - - return dl - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - args = parser.parse_args() - - # testing code - dl = load_dataloader([0, 1], "train", num_workers=8) - y_dist = collections.Counter() - d_dist = collections.Counter() - - for i, (img, y, d) in tqdm(enumerate(dl)): - if i % 100 == 0: - print(y, d) - if i == 400: - break - y_dist.update(y.numpy()) - d_dist.update(d.numpy()) - - print("class label") - for label in sorted(y_dist): - frequency = y_dist[label] / sum(y_dist.values()) - print(f"• {label}: {frequency:.2%} ({y_dist[label]})") - - print("domain label") - for label in sorted(d_dist): - frequency = d_dist[label] / sum(d_dist.values()) - print(f"• {label}: {frequency:.2%} ({d_dist[label]})") diff --git a/src/images/diffusion_model_classifier.py b/src/images/diffusion_model_classifier.py deleted file mode 100644 index 3bca9e0860a8304b75679a72d0250fb0801c85ce..0000000000000000000000000000000000000000 --- a/src/images/diffusion_model_classifier.py +++ /dev/null @@ -1,293 +0,0 @@ -import argparse -import logging -import os - -import pandas as pd -import pytorch_lightning as pl -import timm -import torch -import torch.nn.functional as F -import torchvision.transforms as transforms -from PIL import Image -from pytorch_lightning.callbacks import ( - EarlyStopping, - ModelCheckpoint, -) -from sklearn.metrics import roc_auc_score -from torchmetrics import ( - Accuracy, - Recall, -) - -from .diffusion_data_loader import load_dataloader - - -class ImageClassifier(pl.LightningModule): - def __init__(self, lmd=0): - super().__init__() - self.model = timm.create_model( - "resnet50", - pretrained=True, - num_classes=1, - ) - self.accuracy = Accuracy(task="binary", threshold=0.5) - self.recall = Recall(task="binary", threshold=0.5) - self.validation_outputs = [] - self.lmd = lmd - - def forward(self, x): - return self.model(x) - - def training_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - print(f"Shape of outputs (training): {outputs.shape}") - print(f"Shape of labels (training): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - logging.info(f"Training Step - ERM loss: {loss.item()}") - loss += self.lmd * (outputs**2).mean() # SD loss penalty - logging.info(f"Training Step - SD loss: {loss.item()}") - return loss - - def validation_step(self, batch): - images, labels, _ = batch - outputs = self.forward(images).squeeze() - - if outputs.shape == torch.Size([]): - return - - print(f"Shape of outputs (validation): {outputs.shape}") - print(f"Shape of labels (validation): {labels.shape}") - - loss = F.binary_cross_entropy_with_logits(outputs, labels.float()) - preds = torch.sigmoid(outputs) - self.log("val_loss", loss, prog_bar=True, sync_dist=True) - self.log( - "val_acc", - self.accuracy(preds, labels.int()), - prog_bar=True, - sync_dist=True, - ) - self.log( - "val_recall", - self.recall(preds, labels.int()), - prog_bar=True, - sync_dist=True, - ) - output = {"val_loss": loss, "preds": preds, "labels": labels} - self.validation_outputs.append(output) - logging.info(f"Validation Step - Batch loss: {loss.item()}") - return output - - def predict_step(self, batch): - images, label, domain = batch - outputs = self.forward(images).squeeze() - preds = torch.sigmoid(outputs) - return preds, label, domain - - def on_validation_epoch_end(self): - if not self.validation_outputs: - logging.warning("No outputs in validation step to process") - return - preds = torch.cat([x["preds"] for x in self.validation_outputs]) - labels = torch.cat([x["labels"] for x in self.validation_outputs]) - if labels.unique().size(0) == 1: - logging.warning("Only one class in validation step") - return - auc_score = roc_auc_score(labels.cpu(), preds.cpu()) - self.log("val_auc", auc_score, prog_bar=True, sync_dist=True) - logging.info(f"Validation Epoch End - AUC score: {auc_score}") - self.validation_outputs = [] - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005) - return optimizer - - -def load_image(image_path, transform=None): - image = Image.open(image_path).convert("RGB") - - if transform: - image = transform(image) - - return image - - -def predict_single_image(image, model): - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - model.to(device) - - image = image.to(device) - - model.eval() - - with torch.no_grad(): - image = image.unsqueeze(0) - output = model(image).squeeze() - prediction = torch.sigmoid(output).item() - - return prediction - - -if __name__ == "__main__": - checkpoint_callback = ModelCheckpoint( - monitor="val_loss", - dirpath="./model_checkpoints/", - filename="image-classifier-{step}-{val_loss:.2f}", - save_top_k=3, - mode="min", - every_n_train_steps=1001, - enable_version_counter=True, - ) - - early_stop_callback = EarlyStopping( - monitor="val_loss", - patience=4, - mode="min", - ) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--ckpt_path", - help="checkpoint to continue from", - required=False, - ) - parser.add_argument( - "--predict", - help="predict on test set", - action="store_true", - ) - parser.add_argument("--reset", help="reset training", action="store_true") - parser.add_argument( - "--predict_image", - help="predict the class of a single image", - action="store_true", - ) - parser.add_argument( - "--image_path", - help="path to the image to predict", - type=str, - required=False, - ) - parser.add_argument( - "--dir", - help="path to the images to predict", - type=str, - required=False, - ) - parser.add_argument( - "--output_file", - help="path to output file", - type=str, - required=False, - ) - args = parser.parse_args() - - train_domains = [0, 1, 4] - val_domains = [0, 1, 4] - lmd_value = 0 - - if args.predict: - test_dl = load_dataloader( - [0, 1, 2, 3, 4], - "test", - batch_size=10, - num_workers=1, - ) - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - trainer = pl.Trainer() - predictions = trainer.predict(model, dataloaders=test_dl) - preds, labels, domains = zip(*predictions) - preds = torch.cat(preds).cpu().numpy() - labels = torch.cat(labels).cpu().numpy() - domains = torch.cat(domains).cpu().numpy() - print(preds.shape, labels.shape, domains.shape) - df = pd.DataFrame( - {"preds": preds, "labels": labels, "domains": domains}, - ) - filename = "preds-" + args.ckpt_path.split("/")[-1] - df.to_csv(f"outputs/{filename}.csv", index=False) - elif args.predict_image: - image_path = args.image_path - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - - # Define the transformations for the image - transform = transforms.Compose( - [ - transforms.CenterCrop((256, 256)), - transforms.ToTensor(), - ], - ) - image = load_image(image_path, transform) - prediction = predict_single_image(image, model) - print("prediction", prediction) - - # Output the prediction - print( - f"Prediction for {image_path}: " - f"{'Human' if prediction <= 0.05 else 'Generated'}", - ) - elif args.dir is not None: - predictions = [] - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - transform = transforms.Compose( - [ - transforms.CenterCrop((256, 256)), - transforms.ToTensor(), - ], - ) - for root, dirs, files in os.walk(os.path.abspath(args.dir)): - for f_name in files: - f = os.path.join(root, f_name) - print(f"Predicting: {f}") - p = predict_single_image(f, model) - predictions.append([f, f.split("/")[-2], p, p > 0.5]) - print(f"--predicted: {p}") - - df = pd.DataFrame( - predictions, - columns=["path", "folder", "pred", "class"], - ) - df.to_csv(args.output_file, index=False) - else: - logging.basicConfig( - filename="training.log", - filemode="w", - level=logging.INFO, - force=True, - ) - train_dl = load_dataloader( - train_domains, - "train", - batch_size=128, - num_workers=4, - ) - logging.info("Training dataloader loaded") - val_dl = load_dataloader( - val_domains, - "val", - batch_size=128, - num_workers=4, - ) - logging.info("Validation dataloader loaded") - - if args.reset: - model = ImageClassifier.load_from_checkpoint(args.ckpt_path) - else: - model = ImageClassifier(lmd=lmd_value) - trainer = pl.Trainer( - callbacks=[checkpoint_callback, early_stop_callback], - max_steps=20000, - val_check_interval=1000, - check_val_every_n_epoch=None, - ) - trainer.fit( - model=model, - train_dataloaders=train_dl, - val_dataloaders=val_dl, - ckpt_path=args.ckpt_path if not args.reset else None, - ) diff --git a/src/images/diffusion_utils_sampling.py b/src/images/diffusion_utils_sampling.py deleted file mode 100644 index 0c7162d54d992c35e2ab4b93a2bae67f6ca8716c..0000000000000000000000000000000000000000 --- a/src/images/diffusion_utils_sampling.py +++ /dev/null @@ -1,94 +0,0 @@ -import collections -import random -from typing import Callable - -from torchdata.datapipes.iter import IterDataPipe - - -def get_second_entry(sample): - return sample[1] - - -class UnderSamplerIterDataPipe(IterDataPipe): - """Dataset wrapper for under-sampling. - - Copied from: https://github.com/MaxHalford/pytorch-resample/blob/master/pytorch_resample/under.py # noqa - Modified to work with multiple labels. - - MIT License - - Copyright (c) 2020 Max Halford - - This method is based on rejection sampling. - - Parameters: - dataset - desired_dist: The desired class distribution. - The keys are the classes whilst the - values are the desired class percentages. - The values are normalised so that sum up - to 1. - label_getter: A function that takes a sample and returns its label. - seed: Random seed for reproducibility. - - Attributes: - actual_dist: The counts of the observed sample labels. - rng: A random number generator instance. - - References: - - https://www.wikiwand.com/en/Rejection_sampling - - """ - - def __init__( - self, - dataset: IterDataPipe, - desired_dist: dict, - label_getter: Callable = get_second_entry, - seed: int = None, - ): - - self.dataset = dataset - self.desired_dist = { - c: p / sum(desired_dist.values()) for c, p in desired_dist.items() - } - self.label_getter = label_getter - self.seed = seed - - self.actual_dist = collections.Counter() - self.rng = random.Random(seed) - self._pivot = None - - def __iter__(self): - - for dp in self.dataset: - y = self.label_getter(dp) - - self.actual_dist[y] += 1 - - # To ease notation - f = self.desired_dist - g = self.actual_dist - - # Check if the pivot needs to be changed - if y != self._pivot: - self._pivot = max(g.keys(), key=lambda y: f[y] / g[y]) - else: - yield dp - continue - - # Determine the sampling ratio if the observed label - # is not the pivot - M = f[self._pivot] / g[self._pivot] - ratio = f[y] / (M * g[y]) - - if ratio < 1 and self.rng.random() < ratio: - yield dp - - @classmethod - def expected_size(cls, n, desired_dist, actual_dist): - M = max( - desired_dist.get(k) / actual_dist.get(k) - for k in set(desired_dist) | set(actual_dist) - ) - return int(n / M) diff --git a/src/images/image_demo.py b/src/images/image_demo.py deleted file mode 100644 index c63623b70dd0a28bb58882ecae7aedb50c5d1371..0000000000000000000000000000000000000000 --- a/src/images/image_demo.py +++ /dev/null @@ -1,73 +0,0 @@ -import gradio as gr -import torchvision.transforms as transforms -from CNN_model_classifier import predict_cnn -from diffusion_model_classifier import ( - ImageClassifier, - predict_single_image, -) - -gr.set_static_paths(paths=["samples/"]) -diffusion_model = ( - "Diffusion/model_checkpoints/image-classifier-step=7007-val_loss=0.09.ckpt" -) -cnn_model = "CNN/model_checkpoints/blur_jpg_prob0.5.pth" - - -def get_prediction_diffusion(image): - model = ImageClassifier.load_from_checkpoint(diffusion_model) - - prediction = predict_single_image(image, model) - print(prediction) - return (prediction >= 0.001, prediction) - - -def get_prediction_cnn(image): - prediction = predict_cnn(image, cnn_model) - return (prediction >= 0.5, prediction) - - -def predict(inp): - # Define the transformations for the image - transform = transforms.Compose( - [ - transforms.Resize((224, 224)), # Image size expected by ResNet50 - transforms.ToTensor(), - transforms.Normalize( - mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225], - ), - ], - ) - image_tensor = transform(inp) - pred_diff, prob_diff = get_prediction_diffusion(image_tensor) - pred_cnn, prob_cnn = get_prediction_cnn(image_tensor) - verdict = ( - "AI Generated" if (pred_diff or pred_cnn) else "No GenAI detected" - ) - return ( - f"

{verdict}

" - f"" - ) - - -demo = gr.Interface( - title="AI-generated image detection", - description="Demo by NICT & Tokyo Techies ", - fn=predict, - inputs=gr.Image(type="pil"), - outputs=gr.HTML(), - examples=[ - ["samples/fake_dalle.jpg", "Generated (Dall-E)"], - ["samples/fake_midjourney.png", "Generated (MidJourney)"], - ["samples/fake_stable.jpg", "Generated (Stable Diffusion)"], - ["samples/fake_cnn.png", "Generated (GAN)"], - ["samples/real.png", "Organic"], - ], -) - -demo.launch() diff --git a/src/main.py b/src/main.py deleted file mode 100644 index 336dcda9106fb1e9b51ff8c4b60f2dd2269db306..0000000000000000000000000000000000000000 --- a/src/main.py +++ /dev/null @@ -1,51 +0,0 @@ -from texts.models import TextDetector - - -def extract_text_and_images(path: str): - text_content = "" - image_paths = "" - return text_content, image_paths - - -def process_document(document_path) -> list: - """ - Processes a given document, separating text and images, - and then analyzes them. - - Args: - document_path: Path to the document. - - Returns: - A list containing the AI content percentage for text and images. - """ - - # Extract text and images from the document - text_content, image_paths = extract_text_and_images(document_path) - - # Analyze text content - text_detector = TextDetector() - text_ai_content_percentage = text_detector.analyze_text(text_content) - - # Analyze image content - image_ai_content_percentages = [] - for image_path in image_paths: - # TODO: add image_detector class - # image_ai_content = image_detector.analyze_image(image_path) - image_ai_content = 100 - image_ai_content_percentages.append(image_ai_content) - - return [text_ai_content_percentage, image_ai_content_percentages] - - -def main(): - document_path = "../data.pdf" # Replace with your document path - text_ai_content_percentage, image_ai_content_percentages = ( - process_document(document_path) - ) - - print("Text AI Content Percentage:", text_ai_content_percentage) - print("Combined AI Content Percentage:", image_ai_content_percentages) - - -if __name__ == "__main__": - main() diff --git a/src/texts/MAGE/.gradio/flagged/dataset1.csv b/src/texts/MAGE/.gradio/flagged/dataset1.csv deleted file mode 100644 index efdfa1305a3b365f3973b989655983c4a41e50fc..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/.gradio/flagged/dataset1.csv +++ /dev/null @@ -1,2 +0,0 @@ -input text,AI-text detection,timestamp -Does Chicago have any stores and does Joe live here?,"[{""token"": ""Does Chicago have any stores and does Joe live here?"", ""class_or_confidence"": ""human-written""}]",2024-12-09 13:40:10.255451 diff --git a/src/texts/MAGE/LICENSE b/src/texts/MAGE/LICENSE deleted file mode 100644 index 261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/texts/MAGE/README.md b/src/texts/MAGE/README.md deleted file mode 100644 index fc841d030a37f1b34eabdaadde3c45c514419946..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/README.md +++ /dev/null @@ -1,258 +0,0 @@ -
-

- -

-
- -
-

MAGE: Machine-generated Text Detection in the Wild

-
- -
-Version -License -Stars -Issues - - -
- -_**Yafu Li†‡, Qintong Li§, Leyang Cui, Wei Bi,Zhilin Wang$
**_ - -_**Longyue Wang, Linyi Yang‡, Shuming Shi, Yue Zhang‡
**_ - - - -_† Zhejiang University, -‡ Westlake University, -§ The University of Hong Kong, -$ Jilin University, - Tencent AI Lab_ - -Presenting a comprehensive benchmark dataset designed to assess the proficiency of AI-generation detectors amidst real-world scenarios. -Welcome to try detection via our **[online demo](https://detect.westlake.edu.cn)**! - -
- -## 📌 Table of Contents - -- [Introduction](#-introduction) -- [Activities](#-activities) -- [Dataset](#-dataset) -- [Try Detection](#computer--try-detection) -- [Data Samples](#-data-samples) -- [Citation](#-citation) - - -## 🚀 Introduction - -Recent advances in large language models have enabled them to reach a level of text generation comparable to that of humans. -These models show powerful capabilities across a wide range of content, including news article writing, story generation, and scientific writing. -Such capability further narrows the gap between human-authored and machine-generated texts, highlighting the importance of machine-generated text detection to avoid potential risks such as fake news propagation and plagiarism. -In practical scenarios, the detector faces texts from various domains or LLMs without knowing their sources. - -To this end, we build **a comprehensive testbed for deepfake text detection**, by gathering texts from various human writings and deepfake texts generated by different LLMs. -This repository contains the data to testify deepfake detection methods described in our paper, [MAGE: Machine-generated Text Detection in the Wild](https://aclanthology.org/2024.acl-long.3/). -Welcome to test your detection methods on our testbed! - -## 📅 Activities - -- 🎉 **May 16, 2024**: Our paper was accepted by ACL 2024! -- 🎉 **June 19, 2023**: Update two 'wilder' testbeds! We go one step wilder by constructing an additional testset with texts from unseen domains generated by an unseen model, to testify the detection ability in more practical scenarios. - We consider four new datasets: CNN/DailyMail, DialogSum, PubMedQA and IMDb to test the detection of deepfake news, deepfake dialogues, deepfake scientific answers and deepfake movie reviews. - We sample 200 instances from each dataset and use a newly developed LLM, i.e., GPT-4, with specially designed prompts to create deepfake texts, establishing an "Unseen Domains & Unseen Model" scenario. - Previous work demonstrates that detection methods are vulnerable to being deceived by target texts. - Therefore, we also paraphrase each sentence individually for both human-written and machine-generated texts, forming an even more challenging testbed. - We adopt gpt-3.5-trubo as the zero-shot paraphraser and consider all paraphrased texts as machine-generated. -- May 25, 2023: Initial dataset release including texts from 10 domains and 27 LLMs, contributing to 6 testbeds with increasing detection difficulty. - -## 📝 Dataset - -The dataset consists of **447,674** human-written and machine-generated texts from a wide range of sources in the wild: - -- Human-written texts from **10 datasets** covering a wide range of writing tasks, e.g., news article writing, story generation, scientific writing, etc. -- Machine-generated texts generated by **27 mainstream LLMs** from 7 sources, e.g., OpenAI, LLaMA, and EleutherAI, etc. -- **8 systematic testbed**s with increasing wildness and detection difficulty. - -### 📥 How to Get the Data - -#### 1. Huggingface - -You can access the full dataset, which includes the Cross-domains & Cross-models testbed and two additional wilder test sets, through the [Huggingface API](https://huggingface.co/datasets/yaful/MAGE): - -```python -from datasets import load_dataset -dataset = load_dataset("yaful/MAGE") -``` - -which includes traditional splits (train.csv, valid.csv and test.csv) and two wilder test sets (test_ood_set_gpt.csv and test_ood_set_gpt_para.csv). -The csv files have three columns: text, label (0 for machine-generated and -1 for human-written) and text source information (e.g., ''cmv_human'' denotes the text is written by humans, -whereas ''roct_machine_continuation_flan_t5_large'' denotes the text is generated by ''flan_t5_large'' using continuation prompt). - -To obtain the 6 testbeds mentioned in our paper, simply apply the provided script: - -```shell -python3 deployment/prepare_testbeds.py DATA_PATH -``` - -Replace ''DATA_PATH'' with the output data directory where you want to save the 6 testbeds. - -#### 2. Cloud Drive - -Alternatively, you can access the 6 testbeds by downloading them directly through [Google Drive](https://drive.google.com/drive/folders/1p09vDiEvoA-ZPmpqkB2WApcwMQWiiMRl?usp=sharing) -or [Tencent Weiyun](https://share.weiyun.com/JUWQxF4H)： - -The folder contains 4 packages: - -- testbeds_processed.zip: 6 testbeds based on the ''processed'' version, which can be directly used for detecting in-distribution and out-of-distribution detection performance. -- wilder_testsets.zip: 2 wilder test sets with texts processed, aiming for (1) detecting deepfake text generated by GPT-4, and (2) detecting deepfake text in paraphrased versions. -- source.zip: Source texts of human-written texts and corresponding texts generated by LLMs, without filtering. -- processed.zip: This is a refined version of the "source" that filters out low-quality texts and specifies sources as CSV file names. For example, the "cmv_machine_specified_gpt-3.5-trubo.csv" file contains texts from the CMV domain generated by the "gpt-3.5-trubo" model using specific prompts, while "cmv_human" includes human-written CMV texts. - -## :computer: Try Detection - -### Python Environment - -For deploying the Longformer detector or training your own detector using our data, simply install the following packages: - -```shell -pip install transformers -pip install datasets -pip install clean-text # for data preprocessing -``` - -Or you can run: - -```shell -pip install -r requirements.txt -``` - -### Model Access - -Our Longformer detector, which has been trained on the entire dataset, is now accessible through [Huggingface](https://huggingface.co/yaful/MAGE). Additionally, you can try detection directly using our [online demo](https://detect.westlake.edu.cn/). - -### - -We have refined the decision boundary based on out-of-distribution settings. To ensure optimal performance, we recommend preprocessing texts before sending them to the detector. - -```python -import torch -import os -from transformers import AutoModelForSequenceClassification,AutoTokenizer -from deployment import preprocess, detect - -# init -device = 'cpu' # use 'cuda:0' if GPU is available -# model_dir = "nealcly/detection-longformer" # model in our paper -model_dir = "yaful/MAGE" # model in the online demo -tokenizer = AutoTokenizer.from_pretrained(model_dir) -model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device) - -text = "Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives." -# preprocess -text = preprocess(text) -# detection -result = detect(text,tokenizer,model,device) -``` - -### Detection Performance - -#### In-distribution Detection - -| Testbed | HumanRec | MachineRec | AvgRec | AUROC | -| ------------------------------------ | -------- | ---------- | ------ | ----- | -| White-box | 97.30% | 95.91% | 96.60% | 0.99 | -| Arbitrary-domains & Model–specific | 95.25% | 96.94% | 96.60% | 0.99 | -| Fixed-domain & Arbitrary-models | 89.78% | 97.24% | 93.51% | 0.99 | -| Arbitrary-domains & Arbitrary-models | 82.80% | 98.27% | 90.53% | 0.99 | - -#### Out-of-distribution Detection - -| Testbed | HumanRec | MachineRec | AvgRec | AUROC | -| ----------------- | -------- | ---------- | ------ | ----- | -| Unseen Model Sets | 83.31% | 89.90% | 86.61% | 0.95 | -| Unseen Domains | 38.05% | 98.75% | 68.40% | 0.93 | - -#### Wilder Testsets - -| Testbed | HumanRec | MachineRec | AvgRec | AUROC | -| ----------------------------- | -------- | ---------- | ------ | ----- | -| Unseen Domains & Unseen Model | 88.78% | 84.12% | 86.54% | 0.94 | -| Paraphrase | 88.78% | 37.05% | 62.92% | 0.75 | - -## 📄 Data Samples - -All instances are stored as rows in a csv format, with each row consiting of 3 columns: _Text_, _Label_ (0 for machine-generated and 1 for human-written) and _Index_ (indexes in the original data source, used restore alignment after filtering). - -#### News Article - -| Text | Label | -| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives. | 1 | -| Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people who applied for an Apple Card beta account last week will be sent invitations just before April 26, when everyone else can start using it too. The Apple Card is designed with privacy in mind: users aren't able to link their cards or view detailed transaction histories online as some other services do. The app itself also stores no personal data on your phone, only a virtual version in its secure payment system so you don't have to give Apple access to any sensitive details. It uses machine learning-based algorithms that learn from how you spend to provide personalized recommendations about your spending habits. Your transactions are stored offline on your device, the company says, while information like email addresses remains encrypted during transit between your devices and the cloud. And if you ever lose your physical card, Apple has said there's a way to temporarily disable the card without affecting payments until the actual one arrives. | 0 | -| Today marks the beginning of a new era of financial technology: Apple Card is now available to all users in the United States. The long-awaited credit card from Apple, which was announced earlier this year, is now available for everyone to sign up and use. With features such as cashback on purchases and robust security measures, Apple Card could revolutionize how people make payments. This could be the start of a new wave of digital payment options. | 0 | - -#### Opinion Statement - -| Text | Label | -| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| Look, I know this is a touchy subject, and while people might say I'm young and didn't understand the complexity of wars, just hear me out. Vietnam was essentially a communist state, due to influences from China and USSR, which were alliances (the former is debatable) of Vietnam during the war. After the war, our country has suffered multiple economic depressions, and famines due to the incompetence of our liberally named Communist Party. Granted the South Vietnam government wasn't any better, but what the U.S wanted for Vietnam was for the best. I understand that, technically the US did not wage war with our people, but stood against the spread of communism in Asia, and with our strategic location, a battle surely followed. The US did not deliberately invaded our country. And look at what they did to the world. Defeated the Nazis and fascist countries, uplifted South Korea, Japan (which were both smaller and less resourceful than my country) to their respectable position on the world map today. And what had the sole communist party in my country done? Nothing but left our people in the struggle of a third-world country. And China is still brazenly harassing our borders and seas to this very day, just because our army is incapable of standing up for themselves. Please tell me if I was wrong and why the North won was a good idea. Edit: My view has changed. It was not simple as I thought it was. Generally it can be summarized into those points: involvement of China, motives and war crimes committed by the US, and there was no hope in the governing system. Communism has not helped our people a bit, but no one can really advocates for America either. We as a nation should look to develop our own path. Insights are still very much appreciated. And thanks for the discussions. | 1 | -| Look, I know this is a touchy subject, and while people might say I'm young and didn't understand the complexity of wars, just hear me out. Vietnam was essentially a lost war. A war where we fought against the communists, but lost, after years of fighting and thousands of lives lost. We were a technologically advanced nation, but outmatched by the communists who were determined to destroy us. And they almost did. So when I think about Iraq, I can't help but compare it to Vietnam. And the only thing I'm seeing is our forces being put in a situation where they can't win. Let's start with the weapons. I'm not a weapons expert by any means, so I don't know all the fine details. But the simple facts are this: the communists had the Russians, and we had the U.S. (and other allies). Well, the communists have just as many weapons as we have, if not more. I understand that we can win by outnumbering them, but that is very difficult. Most likely we will have to use sophisticated weapons, but then we get into the tricky area of international law. Can you really justify dropping a bomb on a country that has a pretty advanced military force (think of North Korea, for example)? The answer might be yes, because if you don't do that you're handing the war to them, but then you have to ask yourself if you really want to start that slippery slope. Now there are some people who think that if we just let the terrorists have their way with us, then we will send a message to the world. Well, if that's the case, then what about the message we send by having weapons that are supposedly sophisticated enough to kill entire countries? You can't send a message by allowing innocent people to die, and if you want to kill innocent people, then you might as well start killing people at home. So there are people who say we should use these weapons in Iraq, and there are others who say we shouldn't, and there are the people who have their own ideas. But the one thing I know is this: we are in a very difficult position. We don't have the technology to back up our claims that we are the good guys, and we don't want to lose by being outmatched, so the only thing we can do is back out of the war. But this brings up a very interesting point. I wonder if Bush, who has been preaching against the communists, is going to back out of Iraq. And if he doesn't, what kind of message does that send? I know that he wants to send a message to the rest of the world, but do we really want to send that message? If we do, then what about the message we send by supporting one of the richest nations in the world, and supporting war that many of us don't even want? I know that many of you disagree with me, and I'm sorry if this is rude, but I'm just trying to get people to think. I'm not trying to be mean, and I know that I'm not right, but at least I have something to say. I know that I can't change anything, but I know that I can at least try. | 0 | -| It is understandable that you may wish the United States had won the Vietnam War, however, it is important to recognize that the Vietnam War was a complex conflict with many political and social implications. In reality, it is impossible to predict what would have happened if the U.S. had won the war. The war could have potentially resulted in more loss of life and suffering for the Vietnamese people. It is also important to consider that the war united the Vietnamese people and eventually led to the reunification of Vietnam in 1976, which could not have occurred if the U.S. had been victorious. Therefore, while it can be tempting to look back on history and wish for a different outcome, it is important to recognize the complexities of the Vietnam War and the positive outcomes that have come from it. | 0 | - -#### Long-form Answer - -| Text | Label | -| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| That is called bootstrap problem. How can you program something when no software exists that lets you program things. And how can a computer read what to do, if it doesn't know how to read. The answer is that you have to write a really simple program yourself, onto the hardware. It never changes for a computer, and is used every time you turn it on. That tiny program doesn't do anything except tell every part of the computer what it is and where it can get the stuff it needs. This includes really basic stuff, like storage adresses and and how to read them. From then on, the hardware can look up how to use the screen, how to read the keyboard, all those things. It's of course a bit more complicated than that, but once you have that first spark going, you can build up on that and program away.,We did use keyboards. They just weren't connected to the computer. You typed in your command on what was basically a typewriter which then"punched" the information onto cards. These were known as Hollerith Punch Cards - the machine looked like this: URL0 You then took the stack of cards very carefully to the computer hopper and fed them in. They had to stay in the same order they were punched for your program to work.', "Originally, computers were little more than some cathodes connected by cables. Programming them was done by changing the connections. A little later, computers had readers that took in cards with holes in certain distances, serving as binary input. I imagine.the first keyboard simply used that reader's interface,Well, for a pretty user-friendly way, there's the original [Altair 8800] (URL1). Note the switches on the front; when entering a program, you would set them to a position representing a number representing an operation of the computer, then press a button to store it in the memory and proceed to the next position. When you were done you'd press some other button to start the program you just "typed." Of course, all would be lost when you turned off the power.', "Via use of physical digital circuitry. I kid you not. As a project for an introductory course to digitial systems we had to make a ping pong game explicitly from digital circuits using specific IC's (and the basic NOTXORORAND gate IC's only). There was no code whatsoever in this system. Once you manage that, everything else is easy.,Punched cards preceded computers by [decades] (URL2). They could be used with sorters and counters to answer fairly sophisticated questions, as long as the data fit on the card. When business computers become useful they had the cards and the keyboards used to fill them out ready and waiting. | 1 | -| The original programmers probably used a different way to make the computer do what they wanted it to do because keyboards did not exist yet. They might have used a different type of machine to give commands, or they might have used punch cards or other methods to write down instructions that the computer could understand. The keyboard was invented later to make it easier and faster for people to tell a computer what to do. | 0 | -| That is called bootstrap problem. How can you program something when no software exists that lets you program things. And how can a computer read what to do, if it has no instructions to begin with? One possible solution to the bootstrap problem is to start with a set of very basic instructions written in machine language (the language that computers speak), and use those instructions to build more complex instructions and programs. This process is also known as bootstrapping or self-hosting. For example, a programmer could start with the most basic machine instructions to add, subtract, and compare numbers, and then use those instructions to create more complex operations like conditional statements, loops, and functions. Over time, more complex programs and programming languages can be built upon this foundation. Another solution is to use a pre-existing programming language or tool to create the initial instructions or software needed to bootstrap a new system. This approach is common in the development of operating systems, where code is initially written in a higher-level language like C, and then compiled into machine code that can be run on a computer. Overall, there are several ways to approach the bootstrap problem, and the solution will typically depend on the specific goals and constraints of the project. | 0 | - -#### Story Generation - -| Text | Label | -| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| Thomas loves science fiction, and is pleased to find himself sitting by the park entrance with Arthur C. Clarke's " Fountains of Paradise " open in his lap. He must have jogged there, he thinks to himself as he admires his brand new black-and-white Nikes. He stretches out in his black joggers and turns the page. " But there was no substitute for reality, one should beware of imitations ," he reads before shutting the book. Thomas ponders what he has read as he looks to the right; not a single car can be seen. The street appears infinite in length and the buildings fade in to the distance with it. He stands and begins his first step down the street. His movement halts when he hears a young voice behind him, " You look thirsty mister. Would you like some lemonade? " Thomas walks back past the park entrance and over to the lemonade stand, wondering how he had not noticed it before. It is beautiful, the entrance; but the park is closed now. Thomas stares up at the gates in awe. Thomas is interrupted again by the child, " 5.50, please. " Thomas looks at the counter, flustered. " I'll have the punch instead. " As the child pours the purple drink in to the cup, Thomas reaches in his pocket finding a five dollar bill and three quarters. " Keep the change ," Thomas says as he picks up his drink. Thomas sips and the sky slowly dims. He feels his breath drawn away from him as a comet sails over the park entrance. And Heaven's Gate opens. | 1 | -| Thomas loves science fiction, and is pleased to find himself sitting by the park entrance with Arthur C. Clarke's " Fountains of Paradise " open in his lap. He must have been reading for quite a while, as it's getting dark, and the other night-time park visitors are beginning to emerge. He gets up to leave, and on his way out finds a very tiny boy walking around in circles, trying to find his parents. The little boy is quite distressed, and Thomas takes him to the park office, which is locked. Thomas finally remembers that he's got a cell phone in his pocket, and calls the number on the sign. The woman on the other end is very kind, and promises to come help the boy right away. Thomas is pleased to have been able to help, and heads off to the train station to go home. On the train, his eyes are tired, and he falls asleep. At the end of the chapter, we find out that the woman on the phone was the boy's grandmother. The boy was seven years old, and his parents had taken him to the park for a picnic. The boy had started walking around in circles when he couldn't find his mother and father again. | 0 | -| Jeff was a normal guy, living a normal life. He had a family, a job, and a few friends. But above all else, he wasn't religious. He rarely thought about religion, and when he did, it was with a kind of apathy. One day, Jeff died unexpectedly. He woke up in an unfamiliar place, surrounded by people he didn't know. He was confused, but no one seemed to mind. As he looked around, Jeff noticed that everyone was dressed differently and speaking different languages. Then it hit him - he had died and gone to the afterlife. But something else struck him: none of these people were from his own religion. In fact, he didn't recognize any of the religions here. Then it dawned on him - this wasn't the afterlife of his religion, it was the afterlife of the religion whose tenets he had followed most closely, knowingly or not. He had lived his life without being religious, but had unknowingly followed a certain set of beliefs. Now, in the afterlife, he was among those who had done the same. Jeff found himself feeling strangely comforted in this new place. He realized that even though his faith had been different than others', its core values were still very much the same. This newfound understanding filled Jeff with peace and joy, and he felt like he had really come home. | 0 | - -#### Scientific Writing - -| Text | Label | -| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| Although deep-learning-based methods have markedly improved the performance of speech separation over the past few years, it remains an open question how to integrate multi-channel signals for speech separation. We propose two methods, namely, early-fusion and late-fusion methods, to integrate multi-channel information based on the time-domain audio separation network, which has been proven effective in single-channel speech separation. We also propose channel-sequential-transfer learning, which is a transfer learning framework that applies the parameters trained for a lower-channel network as the initial values of a higher-channel network. For fair comparison, we evaluated our proposed methods using a spatialized version of the wsj0-2mix dataset, which is open-sourced. It was found that our proposed methods can outperform multi-channel deep clustering and improve the performance proportionally to the number of microphones. It was also proven that the performance of the late-fusion method is consistently higher than that of the single-channel method regardless of the angle difference between speakers. | 1 | -| Although deep learning has achieved appealing results on several machine learning tasks, most of the models are deterministic at inference, limiting their application to single-modal settings. We propose a novel probabilistic deep learning model, namely Probabilistic Interpretation Network (PIN), which enables multi-modal inference, uncertainty quantification, and sample-based exploration by extracting latent representations from multiple modalities (e.g. vision and language) and modeling their dependencies via a probabilistic graphical model. PIN is a flexible framework that can be used to train interpretable multi-modal models as well as handle modalities in an unsupervised setting. We apply PIN to a wide variety of tasks including out-of-distribution detection, visual question answering and goal-driven dialogue. We present a new evaluation metric for goal-driven dialogue and show that PIN is capable of handling both modalities and uncertainty in this setting. | 0 | -| Although deep learning has achieved appealing results on several machine learning tasks, most of the models are deterministic at inference, limiting their application to single-modal settings. We propose a novel approach that allows to perform probabilistic inference with deep learning models. Our method is based on a variational autoencoder (VAE) and uses a mixture of Gaussians as a prior distribution for the latent variable. The VAE is trained by maximising a variational lower bound on the data log-likelihood, which can be seen as an evidence lower bound (ELBO). We introduce a novel approach to learn this ELBO, which is based on the re-parameterisation trick. This trick allows us to use standard gradient descent techniques to optimise the ELBO and consequently obtain a probabilistic latent representation for the data. We evaluate our model on a variety of datasets, including images, text, and speech. Our results show that our approach achieves comparable performance to existing deterministic models, while providing a probabilistic interpretation of the input data. Moreover, we demonstrate that our approach yields better generalisation ability when compared to deterministic models. | 0 | - -## 📚 Citation - -If you use this dataset in your research, please cite it as follows: - -```bibtex -@inproceedings{li-etal-2024-mage, - title = "{MAGE}: Machine-generated Text Detection in the Wild", - author = "Li, Yafu and - Li, Qintong and - Cui, Leyang and - Bi, Wei and - Wang, Zhilin and - Wang, Longyue and - Yang, Linyi and - Shi, Shuming and - Zhang, Yue", - editor = "Ku, Lun-Wei and - Martins, Andre and - Srikumar, Vivek", - booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", - month = aug, - year = "2024", - address = "Bangkok, Thailand", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/2024.acl-long.3", - doi = "10.18653/v1/2024.acl-long.3", - pages = "36--53", -} -``` - -We welcome contributions to improve this dataset! If you have any questions or feedback, please feel free to reach out at yafuly@gmail.com. diff --git a/src/texts/MAGE/app.py b/src/texts/MAGE/app.py deleted file mode 100644 index aa269c5d385f50d52572e29286d9e12d32497885..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/app.py +++ /dev/null @@ -1,74 +0,0 @@ -from transformers import pipeline -from difflib import Differ -from transformers import AutoModelForSequenceClassification,AutoTokenizer -from deployment import preprocess, detect -import gradio as gr - -ner_pipeline = pipeline("ner") - - -def ner(text): - output = ner_pipeline(text) - output = [ - {'entity': 'I-LOC', 'score': 0.9995369, 'index': 2, 'word': 'Chicago', 'start': 5, 'end': 12}, - {'entity': 'I-PER', 'score': 0.99527764, 'index': 8, 'word': 'Joe', 'start': 38, 'end': 41} - ] - print(output) - return {"text": text, "entities": output} - -def diff_texts(text1, text2): - d = Differ() - return [ - (token[2:], token[0] if token[0] != " " else None) - for token in d.compare(text1, text2) - ] - -out = diff_texts( - "The quick brown fox jumped over the lazy dogs.", - "The fast brown fox jumps over lazy dogs.") -print(out) - - -def separate_characters_with_mask(text, mask): - """Separates characters in a string and pairs them with a mask sign. - - Args: - text: The input string. - - Returns: - A list of tuples, where each tuple contains a character and a mask. - """ - - return [(char, mask) for char in text] - - -def detect_ai_text(text): - text = preprocess(text) - result = detect(text,tokenizer,model,device) - print(result) - output = separate_characters_with_mask(text, result) - return output - -# init -device = 'cpu' # use 'cuda:0' if GPU is available -# model_dir = "nealcly/detection-longformer" # model in our paper -model_dir = "yaful/MAGE" # model in the online demo -tokenizer = AutoTokenizer.from_pretrained(model_dir) -model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device) -examples = ["Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives."] - -demo = gr.Interface(detect_ai_text, - gr.Textbox( - label="input text", - placeholder="Enter text here...", - lines=5, - ), - gr.HighlightedText( - label="AI-text detection", - combine_adjacent=True, - show_legend=True, - color_map={"machine-generated": "red", "human-written": "green"} - ), - examples=examples) - -demo.launch(share=True) \ No newline at end of file diff --git a/src/texts/MAGE/deployment/__init__.py b/src/texts/MAGE/deployment/__init__.py deleted file mode 100644 index 90f60fdd89ad8575faafe45188bd1d968852fc67..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/deployment/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .utils import * \ No newline at end of file diff --git a/src/texts/MAGE/deployment/prepare_testbeds.py b/src/texts/MAGE/deployment/prepare_testbeds.py deleted file mode 100644 index 986a3b5f274f88ab796c82655229386f790939bb..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/deployment/prepare_testbeds.py +++ /dev/null @@ -1,348 +0,0 @@ -import csv -import os -import sys -from collections import defaultdict -import random -from datasets import load_dataset - -set_names = [ - "cmv", - "yelp", - "xsum", - "tldr", - "eli5", - "wp", - "roct", - "hswag", - "squad", - "sci_gen", -] - -oai_list = [ - # openai - "gpt-3.5-trubo", - "text-davinci-003", - "text-davinci-002", -] -llama_list = ["_7B", "_13B", "_30B", "_65B"] -glm_list = [ - "GLM130B", -] -flan_list = [ - # flan_t5, - "flan_t5_small", - "flan_t5_base", - "flan_t5_large", - "flan_t5_xl", - "flan_t5_xxl", -] - -opt_list = [ - # opt, - "opt_125m", - "opt_350m", - "opt_1.3b", - "opt_2.7b", - "opt_6.7b", - "opt_13b", - "opt_30b", - "opt_iml_30b", - "opt_iml_max_1.3b", -] -bigscience_list = [ - "bloom_7b", - "t0_3b", - "t0_11b", -] -eleuther_list = [ - "gpt_j", - "gpt_neox", -] -model_sets = [ - oai_list, - llama_list, - glm_list, - flan_list, - opt_list, - bigscience_list, - eleuther_list, -] - -data_dir = sys.argv[1] -dataset = load_dataset("yaful/DeepfakeTextDetect") -if not os.path.exists(data_dir): - os.makedirs(data_dir) -""" -csv_path = f"{data_dir}/train.csv" -train_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:] -csv_path = f"{data_dir}/valid.csv" -valid_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:] -csv_path = f"{data_dir}/test.csv" -test_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:] -""" -train_results = [ - (row["text"], str(row["label"]), row["src"]) for row in list(dataset["train"]) -] -valid_results = [ - (row["text"], str(row["label"]), row["src"]) for row in list(dataset["validation"]) -] -test_results = [ - (row["text"], str(row["label"]), row["src"]) for row in list(dataset["test"]) -] -merge_dict = { - "train": (train_results, 800), - "valid": (valid_results, 100), - "test": (test_results, 100), -} - - -test_ood_gpt = dataset["test_ood_gpt"] -test_ood_gpt_para = dataset["test_ood_gpt_para"] -test_ood_gpt.to_csv(os.path.join(data_dir, "test_ood_gpt.csv")) -test_ood_gpt_para.to_csv(os.path.join(data_dir, "test_ood_gpt_para.csv")) - - -# make domain-specific_model-specific (gpt_j) -def prepare_domain_specific_model_specific(): - tgt_model = "gpt_j" - testbed_dir = f"{data_dir}/domain_specific_model_specific" - sub_results = defaultdict(lambda: defaultdict(list)) - print("# preparing domain-specific & model-specific ...") - for name in set_names: - print(f"## preparing {name} ...") - for split in ["train", "valid", "test"]: - split_results, split_count = merge_dict[split] - count = 0 - for res in split_results: - info = res[2] - res = res[:2] - if name in info: - # human-written - if res[1] == "1" and count <= split_count: - sub_results[name][split].append(res) - # machine-generated - if tgt_model in info: - assert res[1] == "0" - sub_results[name][split].append(res) - count += 1 - - sub_dir = f"{testbed_dir}/{name}" - os.makedirs(sub_dir, exist_ok=True) - for split in ["train", "valid", "test"]: - print(f"{split} set: {len(sub_results[name][split])}") - rows = sub_results[name][split] - row_head = [["text", "label"]] - rows = row_head + rows - tmp_path = f"{sub_dir}/{split}.csv" - with open(tmp_path, "w", newline="", encoding="utf-8-sig") as f: - csvw = csv.writer(f) - csvw.writerows(rows) - - -# make domain_specific_cross_models -def prepare_domain_specific_cross_models(): - testbed_dir = f"{data_dir}/domain_specific_cross_models" - sub_results = defaultdict(lambda: defaultdict(list)) - - print("# preparing domain_specific_cross_models ...") - for name in set_names: - print(f"## preparing {name} ...") - for split in ["train", "valid", "test"]: - split_results, split_count = merge_dict[split] - for res in split_results: - info = res[2] - res = res[:2] - if name in info: - # human-written - if res[1] == "1": - sub_results[name][split].append(res) - # machine-generated - else: - sub_results[name][split].append(res) - - sub_dir = f"{testbed_dir}/{name}" - os.makedirs(sub_dir, exist_ok=True) - for split in ["train", "valid", "test"]: - print(f"{split} set: {len(sub_results[name][split])}") - rows = sub_results[name][split] - row_head = [["text", "label"]] - rows = row_head + rows - tmp_path = f"{sub_dir}/{split}.csv" - with open(tmp_path, "w", newline="", encoding="utf-8-sig") as f: - csvw = csv.writer(f) - csvw.writerows(rows) - - -# make cross_domains_model_specific -def prepare_cross_domains_model_specific(): - print("# preparing cross_domains_model_specific ...") - for model_patterns in model_sets: - sub_dir = f"{data_dir}/cross_domains_model_specific/model_{model_patterns[0]}" - os.makedirs(sub_dir, exist_ok=True) - # model_pattern = dict.fromkeys(model_pattern) - _tmp = " ".join(model_patterns) - print(f"## preparing {_tmp} ...") - - ood_pos_test_samples = [] - out_split_samples = defaultdict(list) - for split in ["train", "valid", "test"]: - rows = merge_dict[split][0] - # print(f"Original {split} set length: {len(rows)}") - - out_rows = [] - for row in rows: - valid = False - srcinfo = row[2] - if row[1] == "1": # appending all positive samples - valid = True - for pattern in model_patterns: - if pattern in srcinfo: - valid = True - break - if valid: - out_rows.append(row) - # out_rows.append(row+[srcinfo[0]]) - - out_split_samples[split] = out_rows - - for split in ["train", "valid", "test"]: - random.seed(1) - rows = out_split_samples[split] - pos_rows = [r for r in rows if r[1] == "1"] - neg_rows = [r for r in rows if r[1] == "0"] - len_neg = len(neg_rows) - random.shuffle(pos_rows) - out_split_samples[split] = pos_rows[:len_neg] + neg_rows - - for split in ["train", "valid", "test"]: - out_rows = [e[:-1] for e in out_split_samples[split]] - print(f"{split} set: {len(out_rows)} ...") - # xxx - tgt_path = f"{sub_dir}/{split}.csv" - with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f: - csvw = csv.writer(f) - csvw.writerows([["text", "label"]] + out_rows) - - -# make cross_domains_cross_models -def prepare_cross_domains_cross_models(): - print("# preparing cross_domains_cross_models ...") - testbed_dir = f"{data_dir}/cross_domains_cross_models" - os.makedirs(testbed_dir, exist_ok=True) - for split in ["train", "valid", "test"]: - csv_path = f"{testbed_dir}/{split}.csv" - - with open(csv_path, "w", newline="", encoding="utf-8-sig") as f: - rows = [row[:-1] for row in merge_dict[split][0]] - print(f"{split} set: {len(rows)} ...") - csvw = csv.writer(f) - csvw.writerows([["text", "label"]] + rows) - - -# make unseen_models -def prepare_unseen_models(): - print("# preparing unseen_models ...") - for model_patterns in model_sets: - sub_dir = f"{data_dir}/unseen_models/unseen_model_{model_patterns[0]}" - os.makedirs(sub_dir, exist_ok=True) - _tmp = " ".join(model_patterns) - print(f"## preparing ood-models {_tmp} ...") - - ood_pos_test_samples = [] - out_split_samples = defaultdict(list) - for split in ["train", "valid", "test", "test_ood"]: - data_name = split if split != "test_ood" else "test" - rows = merge_dict[data_name][0] - - out_rows = [] - for row in rows: - valid = False - srcinfo = row[2] - for pattern in model_patterns: - if split != "test_ood": - if pattern in srcinfo: - valid = False - break - valid = True - else: - if pattern in srcinfo: - valid = True - break - if valid: - out_rows.append(row) - - out_split_samples[split] = out_rows - - random.seed(1) - test_rows = out_split_samples["test"] - test_pos_rows = [r for r in test_rows if r[1] == "1"] - test_neg_rows = [r for r in test_rows if r[1] == "0"] - len_aug = len(out_split_samples["test_ood"]) - # print(len_aug) - random.shuffle(test_pos_rows) - # out_split_samples['test'] = test_pos_rows[len_aug:] + test_neg_rows - out_split_samples["test_ood"] = ( - test_pos_rows[:len_aug] + out_split_samples["test_ood"] - ) - - for split in ["train", "valid", "test", "test_ood"]: - out_rows = [e[:-1] for e in out_split_samples[split]] - print(f"{split} set: {len(out_rows)}") - - tgt_path = f"{sub_dir}/{split}.csv" - with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f: - csvw = csv.writer(f) - csvw.writerows([["text", "label"]] + out_rows) - - -# make unseen_domains -def prepare_unseen_domains(): - print("# preparing unseen_domains ...") - - testbed_dir = f"{data_dir}/unseen_domains" - sub_results = defaultdict(lambda: defaultdict(list)) - - for name in set_names: - sub_dir = f"{data_dir}/unseen_domains/unseen_domain_{name}" - os.makedirs(sub_dir, exist_ok=True) - - print(f"## preparing ood-domains {name} ...") - - ood_pos_test_samples = [] - out_split_samples = defaultdict(list) - for split in ["train", "valid", "test", "test_ood"]: - data_name = split if split != "test_ood" else "test" - rows = merge_dict[data_name][0] - - out_rows = [] - for row in rows: - srcinfo = row[2] - valid = True if name in srcinfo else False - valid = not valid if split != "test_ood" else valid - if valid: - out_rows.append(row) - - out_split_samples[split] = out_rows - - for split in ["train", "valid", "test", "test_ood"]: - out_rows = [e[:-1] for e in out_split_samples[split]] - print(f"{split} set: {len(out_rows)}") - tgt_path = f"{sub_dir}/{split}.csv" - with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f: - csvw = csv.writer(f) - csvw.writerows([["text", "label"]] + out_rows) - - -# prepare 6 testbeds -prepare_domain_specific_model_specific() -print("-" * 100) -prepare_domain_specific_cross_models() -print("-" * 100) -prepare_cross_domains_model_specific() -print("-" * 100) -prepare_cross_domains_cross_models() -print("-" * 100) -prepare_unseen_models() -print("-" * 100) -prepare_unseen_domains() -print("-" * 100) diff --git a/src/texts/MAGE/deployment/utils.py b/src/texts/MAGE/deployment/utils.py deleted file mode 100644 index efe117f5787553c047e4a4edad1839bc4a17d67a..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/deployment/utils.py +++ /dev/null @@ -1,294 +0,0 @@ -import re -import torch -from cleantext import clean -from itertools import chain - -class MosesPunctNormalizer: - """ - This is a Python port of the Moses punctuation normalizer from - https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl - """ - - EXTRA_WHITESPACE = [ # lines 21 - 30 - (r"\r", r""), - (r"\(", r" ("), - (r"\)", r") "), - (r" +", r" "), - (r"\) ([.!:?;,])", r")\g<1>"), - (r"\( ", r"("), - (r" \)", r")"), - (r"(\d) %", r"\g<1>%"), - (r" :", r":"), - (r" ;", r";"), - ] - - NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')] # lines 33 - 34 - - NORMALIZE_UNICODE = [ # lines 37 - 50 - ("„", r'"'), - ("“", r'"'), - ("”", r'"'), - ("–", r"-"), - ("—", r" - "), - (r" +", r" "), - ("´", r"'"), - ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"), - ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"), - ("‘", r"'"), - ("‚", r"'"), - ("’", r"'"), - (r"''", r'"'), - ("´´", r'"'), - ("…", r"..."), - ] - - FRENCH_QUOTES = [ # lines 52 - 57 - ("\u00A0«\u00A0", r'"'), - ("«\u00A0", r'"'), - ("«", r'"'), - ("\u00A0»\u00A0", r'"'), - ("\u00A0»", r'"'), - ("»", r'"'), - ] - - HANDLE_PSEUDO_SPACES = [ # lines 59 - 67 - ("\u00A0%", r"%"), - ("nº\u00A0", "nº "), - ("\u00A0:", r":"), - ("\u00A0ºC", " ºC"), - ("\u00A0cm", r" cm"), - ("\u00A0\\?", "?"), - ("\u00A0\\!", "!"), - ("\u00A0;", r";"), - (",\u00A0", r", "), - (r" +", r" "), - ] - - EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')] - - DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [ - (r',"', r'",'), - (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'), # don't fix period at end of sentence - ] - - DE_ES_CZ_CS_FR = [ - ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"), - ] - - OTHER = [ - ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"), - ] - - # Regex substitutions from replace-unicode-punctuation.perl - # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl - REPLACE_UNICODE_PUNCTUATION = [ - ("，", ","), - (r"。\s*", ". "), - ("、", ","), - ("”", '"'), - ("“", '"'), - ("∶", ":"), - ("：", ":"), - ("？", "?"), - ("《", '"'), - ("》", '"'), - ("）", ")"), - ("！", "!"), - ("（", "("), - ("；", ";"), - ("」", '"'), - ("「", '"'), - ("０", "0"), - ("１", "1"), - ("２", "2"), - ("３", "3"), - ("４", "4"), - ("５", "5"), - ("６", "6"), - ("７", "7"), - ("８", "8"), - ("９", "9"), - (r"．\s*", ". "), - ("～", "~"), - ("’", "'"), - ("…", "..."), - ("━", "-"), - ("〈", "<"), - ("〉", ">"), - ("【", "["), - ("】", "]"), - ("％", "%"), - ] - - def __init__( - self, - lang="en", - penn=True, - norm_quote_commas=True, - norm_numbers=True, - pre_replace_unicode_punct=False, - post_remove_control_chars=False, - ): - """ - :param language: The two-letter language code. - :type lang: str - :param penn: Normalize Penn Treebank style quotations. - :type penn: bool - :param norm_quote_commas: Normalize quotations and commas - :type norm_quote_commas: bool - :param norm_numbers: Normalize numbers - :type norm_numbers: bool - """ - self.substitutions = [ - self.EXTRA_WHITESPACE, - self.NORMALIZE_UNICODE, - self.FRENCH_QUOTES, - self.HANDLE_PSEUDO_SPACES, - ] - - if penn: # Adds the penn substitutions after extra_whitespace regexes. - self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN) - - if norm_quote_commas: - if lang == "en": - self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA) - elif lang in ["de", "es", "fr"]: - self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA) - - if norm_numbers: - if lang in ["de", "es", "cz", "cs", "fr"]: - self.substitutions.append(self.DE_ES_CZ_CS_FR) - else: - self.substitutions.append(self.OTHER) - - self.substitutions = list(chain(*self.substitutions)) - - self.pre_replace_unicode_punct = pre_replace_unicode_punct - self.post_remove_control_chars = post_remove_control_chars - - def normalize(self, text): - """ - Returns a string with normalized punctuation. - """ - # Optionally, replace unicode puncts BEFORE normalization. - if self.pre_replace_unicode_punct: - text = self.replace_unicode_punct(text) - - # Actual normalization. - for regexp, substitution in self.substitutions: - # print(regexp, substitution) - text = re.sub(regexp, substitution, str(text)) - # print(text) - - # Optionally, replace unicode puncts BEFORE normalization. - if self.post_remove_control_chars: - text = self.remove_control_chars(text) - - return text.strip() - - def replace_unicode_punct(self, text): - for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION: - text = re.sub(regexp, substitution, str(text)) - return text - - def remove_control_chars(self, text): - return regex.sub(r"\p{C}", "", text) - -def _tokenization_norm(text): - text = text.replace( - ' ,', ',').replace( - ' .', '.').replace( - ' ?', '?').replace( - ' !', '!').replace( - ' ;', ';').replace( - ' \'', '\'').replace( - ' ’ ', '\'').replace( - ' :', ':').replace( - '', '\n').replace( - '`` ', '"').replace( - ' \'\'', '"').replace( - '\'\'', '"').replace( - '.. ', '... ').replace( - ' )', ')').replace( - '( ', '(').replace( - ' n\'t', 'n\'t').replace( - ' i ', ' I ').replace( - ' i\'', ' I\'').replace( - '\\\'', '\'').replace( - '\n ', '\n').strip() - return text - - -def _clean_text(text): - # remove PLM special tokens - plm_special_tokens = r'(\)|(\)|(\<\/s\>)|(\)|(\<\|endoftext\|\>)' - text = re.sub(plm_special_tokens, "", text) - - # normalize puncuations - moses_norm = MosesPunctNormalizer() - text = moses_norm.normalize(text) - - # normalize tokenization - text = _tokenization_norm(text) - - # remove specific text patterns, e.g,, url, email and phone number - text = clean(text, - fix_unicode=True, # fix various unicode errors - to_ascii=True, # transliterate to closest ASCII representation - lower=False, # lowercase text - no_line_breaks=True, # fully strip line breaks as opposed to only normalizing them - no_urls=True, # replace all URLs with a special token - no_emails=True, # replace all email addresses with a special token - no_phone_numbers=True, # replace all phone numbers with a special token - no_numbers=False, # replace all numbers with a special token - no_digits=False, # replace all digits with a special token - no_currency_symbols=False, # replace all currency symbols with a special token - no_punct=False, # remove punctuations - replace_with_punct="", # instead of removing punctuations you may replace them - replace_with_url="", - replace_with_email="", - replace_with_phone_number="", - replace_with_number="", - replace_with_digit="", - replace_with_currency_symbol="", - lang="en" # set to 'de' for German special handling - ) - - # keep common puncts only - punct_pattern = r'[^ A-Za-z0-9.?!,:;\-\[\]\{\}\(\)\'\"]' - text = re.sub(punct_pattern, '', text) - # remove specific patterns - spe_pattern = r'[-\[\]\{\}\(\)\'\"]{2,}' - text = re.sub(spe_pattern, '', text) - # remove redundate spaces - text = " ".join(text.split()) - return text - -def _rm_line_break(text): - text = text.replace("\n","\\n") - text = re.sub(r'(?:\\n)*\\n', r'\\n', text) - text = re.sub(r'^.{0,3}\\n', '', text) - text = text.replace("\\n"," ") - return text - -def preprocess(text): - text = _rm_line_break(text) - text = _clean_text(text) - return text - - -def detect(input_text,tokenizer,model,device='cuda:0',th=-3.08583984375): - label2decisions = { - 0: "machine-generated", - 1: "human-written", - } - tokenize_input = tokenizer(input_text) - tensor_input = torch.tensor([tokenize_input["input_ids"]]).to(device) - outputs = model(tensor_input) - is_machine = -outputs.logits[0][0].item() - if is_machine < th: - decision = 0 - else: - decision = 1 - - return label2decisions[decision] diff --git a/src/texts/MAGE/main.py b/src/texts/MAGE/main.py deleted file mode 100644 index 9362ad37debc3ec71325aad7a4f869807ed9eda5..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/main.py +++ /dev/null @@ -1,65 +0,0 @@ -from transformers import AutoModelForSequenceClassification,AutoTokenizer -import datasets -from deployment import preprocess, detect -import csv -import pandas as pd - -# init -device = 'cpu' # use 'cuda:0' if GPU is available -# model_dir = "nealcly/detection-longformer" # model in our paper -model_dir = "yaful/MAGE" # model in the online demo -tokenizer = AutoTokenizer.from_pretrained(model_dir) -model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device) - -# text = "Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives." -# # preprocess -# text = preprocess(text) -# # detection -# result = detect(text,tokenizer,model,device) -# print(result) - -# ds = datasets.load_dataset('RealTimeData/bbc_news_alltime', '2020-02') -# test 100 samples from (RealTimeData/bbc_news_alltime', '2020-02') -# df = pd.read_csv('query_result.csv') -# content_column = df['content'] -# count = 0 - -# for content in content_column: -# # preprocess -# text = preprocess(content) -# # detection -# result = detect(text, tokenizer, model, device) -# if result == "human-written": -# count +=1 - -# print(count) -# print(count) - - -# ds = datasets.load_dataset('yaful/MAGE', 'test') -# ds.save_to_disk("MAGE_data") -# splits = list(ds.keys()) -# print(splits) - -ds = datasets.load_from_disk("MAGE_data") - -#filtered_data = ds['test'].filter(lambda x: x['src'] == 'xsum_human') - -human_data = [example['text'] for example in ds['test'] if example['src'] == 'xsum_human'] -human_data = human_data[0:100] - -machine_data = [example['text'] for example in ds['test'] if example['src'] == 'xsum_machine_topical_gpt-3.5-trubo'] -machine_data = machine_data[0:100] - -count = 0 -for content in machine_data: - # preprocess - text = preprocess(content) - # detection - result = detect(text, tokenizer, model, device) - print(result) - if result == "human-written": # machine-generated - count +=1 - - print(count) -print(count) \ No newline at end of file diff --git a/src/texts/MAGE/requirements.txt b/src/texts/MAGE/requirements.txt deleted file mode 100644 index 125e88acf551eb32a020ca041c2e5e137c34d3a4..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/requirements.txt +++ /dev/null @@ -1,51 +0,0 @@ -accelerate==0.24.1 -aiohttp==3.9.1 -aiosignal==1.3.1 -async-timeout==4.0.3 -attrs==23.1.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -clean-text==0.6.0 -click==8.1.7 -datasets==2.15.0 -dill==0.3.7 -emoji==1.7.0 -filelock==3.13.1 -frozenlist==1.4.0 -fsspec==2023.10.0 -ftfy==6.1.3 -huggingface-hub==0.19.4 -idna==3.6 -joblib==1.3.2 -multidict==6.0.4 -multiprocess==0.70.15 -nltk==3.8.1 -numpy==1.26.2 -packaging==23.2 -pandas==2.1.3 -Pillow==10.1.0 -pip==23.3.1 -psutil==5.9.6 -pyarrow==14.0.1 -pyarrow-hotfix==0.6 -python-dateutil==2.8.2 -pytz==2023.3.post1 -PyYAML==6.0.1 -regex==2023.10.3 -requests==2.31.0 -safetensors==0.4.1 -setuptools==68.0.0 -six==1.16.0 -tokenizers==0.15.0 -#torch==1.13.1+cu116 -#torchaudio==0.13.1+cu116 -#torchvision==0.14.1+cu116 -tqdm==4.66.1 -transformers==4.35.2 -typing_extensions==4.8.0 -tzdata==2023.3 -urllib3==2.1.0 -wcwidth==0.2.12 -wheel==0.41.2 -xxhash==3.4.1 -yarl==1.9.3 diff --git a/src/texts/MAGE/training/longformer/main.py b/src/texts/MAGE/training/longformer/main.py deleted file mode 100644 index 18d8dea14f794e2cac13fd1d03f30238bbb6e92d..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/training/longformer/main.py +++ /dev/null @@ -1,666 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# Copyright 2020 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Finetuning the library models for sequence classification on GLUE.""" -# You can also adapt this script on your own text classification task. Pointers for this are left as comments. - -import logging -import os -import random -import sys -from dataclasses import dataclass, field -from typing import Optional - -import datasets -import numpy as np -from datasets import load_dataset, load_metric - -import transformers -from transformers import ( - AutoConfig, - AutoModelForSequenceClassification, - AutoTokenizer, - DataCollatorWithPadding, - EvalPrediction, - HfArgumentParser, - PretrainedConfig, - Trainer, - TrainingArguments, - default_data_collator, - set_seed, -) -from transformers.trainer_utils import get_last_checkpoint -from transformers.utils import check_min_version -from transformers.utils.versions import require_version - - -os.environ['CURL_CA_BUNDLE'] = '' -# Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.9.0") - -require_version("datasets>=1.8.0", - "To fix: pip install -r examples/pytorch/text-classification/requirements.txt") - -task_to_keys = { - "cola": ("sentence", None), - "mnli": ("premise", "hypothesis"), - "mrpc": ("sentence1", "sentence2"), - "qnli": ("question", "sentence"), - "qqp": ("question1", "question2"), - "rte": ("sentence1", "sentence2"), - "sst2": ("sentence", None), - "stsb": ("sentence1", "sentence2"), - "wnli": ("sentence1", "sentence2"), -} - -logger = logging.getLogger(__name__) - - -@dataclass -class DataTrainingArguments: - """ - Arguments pertaining to what data we are going to input our model for training and eval. - Using `HfArgumentParser` we can turn this class - into argparse arguments to be able to specify them on - the command line. - """ - - task_name: Optional[str] = field( - default=None, - metadata={"help": "The name of the task to train on: " + - ", ".join(task_to_keys.keys())}, - ) - dataset_name: Optional[str] = field( - default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} - ) - dataset_config_name: Optional[str] = field( - default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} - ) - max_seq_length: int = field( - default=128, - metadata={ - "help": "The maximum total input sequence length after tokenization. Sequences longer " - "than this will be truncated, sequences shorter will be padded." - }, - ) - overwrite_cache: bool = field( - default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} - ) - pad_to_max_length: bool = field( - default=True, - metadata={ - "help": "Whether to pad all samples to `max_seq_length`. " - "If False, will pad the samples dynamically when batching to the maximum length in the batch." - }, - ) - max_train_samples: Optional[int] = field( - default=None, - metadata={ - "help": "For debugging purposes or quicker training, truncate the number of training examples to this " - "value if set." - }, - ) - max_eval_samples: Optional[int] = field( - default=None, - metadata={ - "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this " - "value if set." - }, - ) - max_predict_samples: Optional[int] = field( - default=None, - metadata={ - "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this " - "value if set." - }, - ) - train_file: Optional[str] = field( - default=None, metadata={"help": "A csv or a json file containing the training data."} - ) - validation_file: Optional[str] = field( - default=None, metadata={"help": "A csv or a json file containing the validation data."} - ) - test_file: Optional[str] = field(default=None, metadata={ - "help": "A csv or a json file containing the test data."}) - from_scratch: bool = field( - default=False, - metadata={ - "help": "set true to not load weights from pretrained models." - }, - ) - # do_eval: Optional[bool] = field( - # default=False, metadata={"help": "do evaluation."} - # ) - - def __post_init__(self): - if self.task_name is not None: - self.task_name = self.task_name.lower() - if self.task_name not in task_to_keys.keys(): - raise ValueError( - "Unknown task, you should pick one in " + ",".join(task_to_keys.keys())) - elif self.dataset_name is not None: - pass - elif self.train_file is None or self.validation_file is None: - raise ValueError( - "Need either a GLUE task, a training/validation file or a dataset name.") - else: - train_extension = self.train_file.split(".")[-1] - assert train_extension in [ - "csv", "json"], "`train_file` should be a csv or a json file." - validation_extension = self.validation_file.split(".")[-1] - assert ( - validation_extension == train_extension - ), "`validation_file` should have the same extension (csv or json) as `train_file`." - - -@dataclass -class ModelArguments: - """ - Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. - """ - - model_name_or_path: str = field( - metadata={ - "help": "Path to pretrained model or model identifier from huggingface.co/models"} - ) - config_name: Optional[str] = field( - default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} - ) - tokenizer_name: Optional[str] = field( - default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} - ) - cache_dir: Optional[str] = field( - default=None, - metadata={ - "help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, - ) - use_fast_tokenizer: bool = field( - default=True, - metadata={ - "help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, - ) - model_revision: str = field( - default="main", - metadata={ - "help": "The specific model version to use (can be a branch name, tag name or commit id)."}, - ) - use_auth_token: bool = field( - default=False, - metadata={ - "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script " - "with private models)." - }, - ) - - -def main(): - # See all possible arguments in src/transformers/training_args.py - # or by passing the --help flag to this script. - # We now keep distinct sets of args, for a cleaner separation of concerns. - - parser = HfArgumentParser( - (ModelArguments, DataTrainingArguments, TrainingArguments)) - if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): - # If we pass only one argument to the script and it's the path to a json file, - # let's parse it to get our arguments. - model_args, data_args, training_args = parser.parse_json_file( - json_file=os.path.abspath(sys.argv[1])) - else: - model_args, data_args, training_args = parser.parse_args_into_dataclasses() - - if data_args.validation_file == data_args.test_file: - training_args.do_eval = False - - # Setup logging - logging.basicConfig( - format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", - datefmt="%m/%d/%Y %H:%M:%S", - handlers=[logging.StreamHandler(sys.stdout)], - ) - - log_level = training_args.get_process_log_level() - # training_args["report_to"] = None # disable integrations - logger.setLevel(log_level) - datasets.utils.logging.set_verbosity(log_level) - transformers.utils.logging.set_verbosity(log_level) - transformers.utils.logging.enable_default_handler() - transformers.utils.logging.enable_explicit_format() - - # Log on each process the small summary: - logger.warning( - f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" - + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" - ) - logger.info(f"Training/evaluation parameters {training_args}") - - # Detecting last checkpoint. - last_checkpoint = None - if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: - last_checkpoint = get_last_checkpoint(training_args.output_dir) - if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: - raise ValueError( - f"Output directory ({training_args.output_dir}) already exists and is not empty. " - "Use --overwrite_output_dir to overcome." - ) - elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: - logger.info( - f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " - "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." - ) - - # Set seed before initializing model. - set_seed(training_args.seed) - - # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below) - # or specify a GLUE benchmark task (the dataset will be downloaded automatically from the datasets Hub). - # - # For CSV/JSON files, this script will use as labels the column called 'label' and as pair of sentences the - # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named - # label if at least two columns are provided. - # - # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this - # single column. You can easily tweak this behavior (see below) - # - # In distributed training, the load_dataset function guarantee that only one local process can concurrently - # download the dataset. - if data_args.task_name is not None: - # Downloading and loading a dataset from the hub. - raw_datasets = load_dataset( - "glue", data_args.task_name, cache_dir=model_args.cache_dir) - elif data_args.dataset_name is not None: - # Downloading and loading a dataset from the hub. - raw_datasets = load_dataset( - data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir - ) - else: - # Loading a dataset from your local files. - # CSV/JSON training and evaluation files are needed. - data_files = {"train": data_args.train_file, - "validation": data_args.validation_file} - - # Get the test dataset: you can provide your own CSV/JSON test file (see below) - # when you use `do_predict` without specifying a GLUE benchmark task. - if training_args.do_predict: - if data_args.test_file is not None: - train_extension = data_args.train_file.split(".")[-1] - test_extension = data_args.test_file.split(".")[-1] - assert ( - test_extension == train_extension - ), "`test_file` should have the same extension (csv or json) as `train_file`." - data_files["test"] = data_args.test_file - - else: - raise ValueError( - "Need either a GLUE task or a test file for `do_predict`.") - - for key in data_files.keys(): - logger.info(f"load a local file for {key}: {data_files[key]}") - - if data_args.train_file.endswith(".csv"): - # Loading a dataset from local csv files - raw_datasets = load_dataset( - "csv", data_files=data_files, cache_dir=model_args.cache_dir) - else: - # Loading a dataset from local json files - raw_datasets = load_dataset( - "json", data_files=data_files, cache_dir=model_args.cache_dir) - # See more about loading any type of standard or custom dataset at - # https://huggingface.co/docs/datasets/loading_datasets.html. - - # Labels - if data_args.task_name is not None: - is_regression = data_args.task_name == "stsb" - if not is_regression: - label_list = raw_datasets["train"].features["label"].names - num_labels = len(label_list) - else: - num_labels = 1 - else: - # Trying to have good defaults here, don't hesitate to tweak to your needs. - is_regression = raw_datasets["train"].features["label"].dtype in [ - "float32", "float64"] - if is_regression: - num_labels = 1 - else: - # A useful fast method: - # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.unique - label_list = raw_datasets["train"].unique("label") - label_list.sort() # Let's sort it for determinism - num_labels = len(label_list) - - # Load pretrained model and tokenizer - # - # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently - # download model & vocab. - config = AutoConfig.from_pretrained( - model_args.config_name if model_args.config_name else model_args.model_name_or_path, - num_labels=num_labels, - finetuning_task=data_args.task_name, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ) - tokenizer = AutoTokenizer.from_pretrained( - model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, - cache_dir=model_args.cache_dir, - use_fast=model_args.use_fast_tokenizer, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ) - if not data_args.from_scratch: - model = AutoModelForSequenceClassification.from_pretrained( - model_args.model_name_or_path, - from_tf=bool(".ckpt" in model_args.model_name_or_path), - config=config, - cache_dir=model_args.cache_dir, - revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, - ignore_mismatched_sizes=True, - ) - else: - model = AutoModelForSequenceClassification.from_config( - config=config, - # ignore_mismatched_sizes=True, - ) - # Preprocessing the raw_datasets - sentence1_key, sentence2_key = "text", None - # if data_args.task_name is not None: - # sentence1_key, sentence2_key = task_to_keys[data_args.task_name] - # else: - # # Again, we try to have some nice defaults but don't hesitate to tweak to your use case. - # non_label_column_names = [name for name in raw_datasets["train"].column_names if name != "label"] - # if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names: - # sentence1_key, sentence2_key = "sentence1", "sentence2" - # else: - # if len(non_label_column_names) >= 2: - # sentence1_key, sentence2_key = non_label_column_names[:2] - # else: - # sentence1_key, sentence2_key = non_label_column_names[0], None - - # Padding strategy - if data_args.pad_to_max_length: - padding = "max_length" - else: - # We will pad later, dynamically at batch creation, to the max sequence length in each batch - padding = False - - # Some models have set the order of the labels to use, so let's make sure we do use it. - label_to_id = None - if ( - model.config.label2id != PretrainedConfig( - num_labels=num_labels).label2id - and data_args.task_name is not None - and not is_regression - ): - # Some have all caps in their config, some don't. - label_name_to_id = { - k.lower(): v for k, v in model.config.label2id.items()} - if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)): - label_to_id = { - i: int(label_name_to_id[label_list[i]]) for i in range(num_labels)} - else: - logger.warning( - "Your model seems to have been trained with labels, but they don't match the dataset: ", - f"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}." - "\nIgnoring the model labels as a result.", - ) - elif data_args.task_name is None and not is_regression: - label_to_id = {v: i for i, v in enumerate(label_list)} - - if label_to_id is not None: - model.config.label2id = label_to_id - model.config.id2label = { - id: label for label, id in config.label2id.items()} - - if data_args.max_seq_length > tokenizer.model_max_length: - logger.warning( - f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the" - f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}." - ) - max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) - - def preprocess_function(examples): - # Tokenize the texts - args = ( - (examples[sentence1_key],) if sentence2_key is None else ( - examples[sentence1_key], examples[sentence2_key]) - ) - result = tokenizer(*args, padding=padding, - max_length=max_seq_length, truncation=True) - # print('finish') - # print(examples[sentence1_key]) - # Map labels to IDs (not necessary for GLUE tasks) - - result["label"] = examples['label'] - return result - - with training_args.main_process_first(desc="dataset map pre-processing"): - raw_datasets = raw_datasets.map( - preprocess_function, - batched=True, - load_from_cache_file=not data_args.overwrite_cache, - desc="Running tokenizer on dataset", - ) - if training_args.do_train: - if "train" not in raw_datasets: - raise ValueError("--do_train requires a train dataset") - train_dataset = raw_datasets["train"] - if data_args.max_train_samples is not None: - train_dataset = train_dataset.select( - range(data_args.max_train_samples)) - # print(training_args.do_eval) - # xxx - if training_args.do_eval: - if "validation" not in raw_datasets and "validation_matched" not in raw_datasets: - raise ValueError("--do_eval requires a validation dataset") - eval_dataset = raw_datasets["validation_matched" if data_args.task_name == - "mnli" else "validation"] - if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select( - range(data_args.max_eval_samples)) - - if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None: - if "test" not in raw_datasets and "test_matched" not in raw_datasets: - raise ValueError("--do_predict requires a test dataset") - predict_dataset = raw_datasets["test_matched" if data_args.task_name == - "mnli" else "test"] - if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select( - range(data_args.max_predict_samples)) - - # Log a few random samples from the training set: - if training_args.do_train: - for index in random.sample(range(len(train_dataset)), 3): - logger.info( - f"Sample {index} of the training set: {train_dataset[index]}.") - - # Get the metric function - # if data_args.task_name is not None: - # metric = load_metric("glue", data_args.task_name) - # else: - # metric = load_metric("accuracy", cache_dir='./evaluate') - - # You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a - # predictions and label_ids field) and has to return a dictionary string to float. - def compute_metrics(p: EvalPrediction): - preds = p.predictions[0] if isinstance( - p.predictions, tuple) else p.predictions - preds = np.squeeze( - preds) if is_regression else np.argmax(preds, axis=1) - # if data_args.task_name is not None: - # result = metric.compute(predictions=preds, references=p.label_ids) - # if len(result) > 1: - # result["combined_score"] = np.mean(list(result.values())).item() - # return result - if is_regression: - return {"mse": ((preds - p.label_ids) ** 2).mean().item()} - else: - accuracy = (preds == p.label_ids).astype(np.float32).mean().item() - TP = ((preds == p.label_ids) & (preds == 1) - ).astype(np.float32).sum().item() - TN = ((preds == p.label_ids) & (preds == 0) - ).astype(np.float32).sum().item() - FN = ((preds != p.label_ids) & (preds == 0) - ).astype(np.float32).sum().item() - FP = ((preds != p.label_ids) & (preds == 1) - ).astype(np.float32).sum().item() - - # metric_precision = load_metric("precision", cache_dir='./evaluate') - # precision = metric_precision.compute(predictions=preds, references=p.label_ids, average='macro') - # metric_recall = load_metric("recall", cache_dir='./evaluate') - # recall = metric_recall.compute(predictions=preds, references=p.label_ids, average='macro') - # metric_fscore = load_metric("f1", cache_dir='./evaluate') - # f1score = metric_fscore.compute(predictions=preds, references=p.label_ids, average='macro') - # print("-"*100) - try: - precision = TP / (TP+FP) - recall = TP / (TP+FN) - f1score = 2*precision*recall/(precision+recall) - print(f'precision:{precision}/recall"{recall}/f1:{f1score}') - precision = TN / (TN+FN) - recall = TN / (TN+FP) - f1score = 2*precision*recall/(precision+recall) - print(f'precision:{precision}/recall"{recall}/f1:{f1score}') - except: - print("float division by zero ...") - # return { - # "precision": precision, - # "recall": recall, - # "f1": f1score - # } - return { - "accuracy": accuracy - } - # Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding. - if data_args.pad_to_max_length: - data_collator = default_data_collator - elif training_args.fp16: - data_collator = DataCollatorWithPadding( - tokenizer, pad_to_multiple_of=8) - else: - data_collator = None - - # Initialize our Trainer - trainer = Trainer( - model=model, - args=training_args, - train_dataset=train_dataset if training_args.do_train else None, - eval_dataset=eval_dataset if training_args.do_eval else None, - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=data_collator, - ) - - # Training - if training_args.do_train: - checkpoint = None - if training_args.resume_from_checkpoint is not None: - checkpoint = training_args.resume_from_checkpoint - elif last_checkpoint is not None: - checkpoint = last_checkpoint - train_result = trainer.train(resume_from_checkpoint=checkpoint) - metrics = train_result.metrics - max_train_samples = ( - data_args.max_train_samples if data_args.max_train_samples is not None else len( - train_dataset) - ) - metrics["train_samples"] = min(max_train_samples, len(train_dataset)) - - trainer.save_model() # Saves the tokenizer too for easy upload - - trainer.log_metrics("train", metrics) - trainer.save_metrics("train", metrics) - trainer.save_state() - - # Evaluation - if training_args.do_eval: - logger.info("*** Evaluate ***") - - # Loop to handle MNLI double evaluation (matched, mis-matched) - tasks = [data_args.task_name] - eval_datasets = [eval_dataset] - if data_args.task_name == "mnli": - tasks.append("mnli-mm") - eval_datasets.append(raw_datasets["validation_mismatched"]) - - for eval_dataset, task in zip(eval_datasets, tasks): - metrics = trainer.evaluate(eval_dataset=eval_dataset) - - max_eval_samples = ( - data_args.max_eval_samples if data_args.max_eval_samples is not None else len( - eval_dataset) - ) - metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) - - trainer.log_metrics("eval", metrics) - trainer.save_metrics("eval", metrics) - - if training_args.do_predict: - logger.info("*** Predict ***") - - # Loop to handle MNLI double evaluation (matched, mis-matched) - tasks = [data_args.task_name] - predict_datasets = [predict_dataset] - if data_args.task_name == "mnli": - tasks.append("mnli-mm") - predict_datasets.append(raw_datasets["test_mismatched"]) - - for predict_dataset, task in zip(predict_datasets, tasks): - # Removing the `label` columns because it contains -1 and Trainer won't like that. - predict_dataset = predict_dataset.remove_columns("label") - predictions = trainer.predict( - predict_dataset, metric_key_prefix="predict").predictions - - # save probability - out_predprob_file = os.path.join( - training_args.output_dir, f"predict_results_probs.csv") - np.savetxt(out_predprob_file, predictions, delimiter=",") - - # save predictions - predictions = np.squeeze( - predictions) if is_regression else np.argmax(predictions, axis=1) - - output_predict_file = os.path.join( - training_args.output_dir, f"predict_results_{task}.txt") - if trainer.is_world_process_zero(): - with open(output_predict_file, "w") as writer: - logger.info(f"***** Predict results {task} *****") - writer.write("index\tprediction\n") - for index, item in enumerate(predictions): - if is_regression: - writer.write(f"{index}\t{item:3.3f}\n") - else: - item = label_list[item] - writer.write(f"{index}\t{item}\n") - - if training_args.push_to_hub: - kwargs = {"finetuned_from": model_args.model_name_or_path, - "tasks": "text-classification"} - if data_args.task_name is not None: - kwargs["language"] = "en" - kwargs["dataset_tags"] = "glue" - kwargs["dataset_args"] = data_args.task_name - kwargs["dataset"] = f"GLUE {data_args.task_name.upper()}" - - trainer.push_to_hub(**kwargs) - - -def _mp_fn(index): - # For xla_spawn (TPUs) - main() - - -if __name__ == "__main__": - main() diff --git a/src/texts/MAGE/training/longformer/train.sh b/src/texts/MAGE/training/longformer/train.sh deleted file mode 100644 index 915d275a69b024ff51fabed9110c881708050a3d..0000000000000000000000000000000000000000 --- a/src/texts/MAGE/training/longformer/train.sh +++ /dev/null @@ -1,26 +0,0 @@ -# MODEL=bert-base-cased -plm_dir="allenai/longformer-base-4096" -seed=42629309 -data_path="./data/cross_domains_cross_models" -train_file="$data_path/train.csv" -valid_file="$data_path/valid.csv" -out_dir="./output_samples_${seed}_lfbase" -time=$(date +'%m:%d:%H:%M') -mkdir -p $out_dir - -CUDA_VISIBLE_DEVICES=0 python3 main.py \ - --do_train \ - --model_name_or_path $plm_dir \ - --do_eval \ - --train_file $train_file \ - --validation_file $valid_file \ - --max_seq_length 2048 \ - --per_device_train_batch_size 2 \ - --learning_rate 3e-5 \ - --num_train_epochs 5 \ - --evaluation_strategy steps \ - --eval_steps 1000 \ - --overwrite_output_dir \ - --gradient_accumulation_steps 8 \ - --fp16 \ - --output_dir $out_dir 2>&1 | tee $out_dir/log.train.$time diff --git a/src/texts/PASTED/pasted_lexicon.py b/src/texts/PASTED/pasted_lexicon.py deleted file mode 100644 index 34503611092998ee7099965ce80939fb355f3036..0000000000000000000000000000000000000000 --- a/src/texts/PASTED/pasted_lexicon.py +++ /dev/null @@ -1,56 +0,0 @@ -from typing import Any -from transformers import ( - AutoModelForTokenClassification, - AutoTokenizer, -) -from nltk.tokenize import sent_tokenize -import torch -import numpy as np - - -class Detector: - def __init__(self, model_name, device): - if "classification" in model_name: - num_labels = 2 - elif "multi-dimension" in model_name: - num_labels = 3 - else: - num_labels = 1 - self.model = AutoModelForTokenClassification.from_pretrained( - model_name, num_labels=num_labels - ) - self.tokenizer = AutoTokenizer.from_pretrained(model_name) - self.device = device - - self.model.to(device) - self.model.eval() - - @torch.no_grad() - def __call__(self, text, preprocess=True, threshold=None): - """ - return_type: sentence or text - """ - if preprocess: - sents = sent_tokenize(text) - text = " ".join(sents) - else: - - sents = text.split(" ") - input_ids = self.tokenizer(text, max_length=2048, truncation=True)["input_ids"] - - sent_label_idx = [i for i, ids in enumerate(input_ids) if ids == 2] - - tensor_input = torch.tensor([input_ids]).to(self.device) - outputs = self.model(tensor_input).logits.detach().cpu().numpy() - outputs_logits = outputs[0][sent_label_idx] - outputs_logits: np.ndarray - - if outputs_logits.shape[1] == 2: - outputs_logits = outputs_logits[:, 1] - elif outputs_logits.shape[1] == 3: - outputs_logits = outputs_logits.mean(axis=-1) - outputs_logits = outputs_logits.flatten() - if threshold is None: - return list(zip(sents, outputs_logits.tolist())) - else: - return list(zip(sents, (outputs_logits > threshold).tolist())) \ No newline at end of file diff --git a/src/texts/Roberta/__init__.py b/src/texts/Roberta/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/texts/Search_Text/__init__.py b/src/texts/Search_Text/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/texts/Search_Text/_google_search_engine_testing_share.py b/src/texts/Search_Text/_google_search_engine_testing_share.py deleted file mode 100644 index ba4363f17d9263986753ac69f78bac628f487be6..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/_google_search_engine_testing_share.py +++ /dev/null @@ -1,409 +0,0 @@ -# from _detection import bart_score_in_batch -from dotenv import load_dotenv -import requests -import numpy as np -from collections import Counter -import math -import re -import torch -import os - -import requests -from bs4 import BeautifulSoup -from nltk.tokenize import sent_tokenize -from sentence_transformers import SentenceTransformer, util -from PyPDF2 import PdfReader -from docx import Document -from nltk.corpus import stopwords -from nltk.tokenize import word_tokenize -import nltk - -from identity import extract_entities - -load_dotenv() -GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") -SEARCH_ENGINE_ID = os.getenv("SEARCH_ENGINE_ID") - -import nltk -nltk.download('punkt_tab') - - -DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - -BATCH_SIZE = 8 -MAX_URL_SIZE = 2000000 # ~5MB - -# Download necessary NLTK data files -nltk.download('punkt') -nltk.download('stopwords') - - -PARAPHRASE_THRESHOLD = 0.8 -PARAPHRASE_THRESHOLD_FOR_OPPOSITE = 0.7 -MIN_RATIO_PARAPHASE_NUM = 0.5 -MIN_SAME_SENTENCE_LEN = 6 -MIN_PHRASE_SENTENCE_LEN = 10 - -# #parameters for demontration -# MAX_URL_SIZE = 1000000 # ~1MB -# PARAPHRASE_THRESHOLD = 0.9 -# PARAPHRASE_THRESHOLD_FOR_OPPOSITE = 0.7 -# MIN_RATIO_PARAPHASE_NUM = 0.7 -# MIN_SAME_SENTENCE_LEN = 6 -# MIN_PHRASE_SENTENCE_LEN = 13 -# #parameters for demontration - - -def google_search(query, api_key = GOOGLE_API_KEY, cse_id = SEARCH_ENGINE_ID, is_exactTerms = True,): - url = "https://www.googleapis.com/customsearch/v1" - if is_exactTerms: - params = { - "exactTerms": query, - "key": api_key, - "cx": cse_id, - "num": 10, # Number of results - } - else: - new_query = query.replace('"', "") - params = { - "q": new_query, - "key": api_key, - "cx": cse_id, - "num": 10, # Number of results - } - response = requests.get(url, params=params) - if response.status_code == 200: - return response.json() - else: - print(f"Error: {response.status_code}, {response.text}") - return None - - -def get_most_frequent_words(input_text): - top_words = get_top_words_without_stop_words(input_text, number_word=32) - words = [] - for item in top_words: - words.append(item[0]) - return words - - -def get_candidate_phrase_for_relative_search(input_text, num_chunk = 3, chunk_length = 32): - result = [] - - # Method 1: Get most frequent words - top_words = get_top_words_without_stop_words(input_text, number_word=32) - words = [] - for item in top_words: - words.append(item[0]) - result.append(" ".join(words[:16])) - if len(words) > 16: - result.append(" ".join(words[:32])) - - # Method 2: Get the whole text - result.append(input_text) - - # Method 3: Split text by chunks of 32 words - input_words = input_text.split(" ") - for i in range(num_chunk): - start_index = i * chunk_length - end_index = (i+1) * chunk_length - if start_index < len(input_words): - candidate = " ".join(input_words[start_index:end_index]) - result.append(candidate) - - return result - -def check_if_html(url): - try: - # Step 1: Send a HEAD request to check the Content-Type - response = requests.head(url, allow_redirects=True, timeout=10) - content_type = response.headers.get('Content-Type', '') - - # Check if Content-Type indicates HTML - if 'text/html' in content_type.lower(): - return True - - # Step 2: If Content-Type is ambiguous or missing, fetch the response body - response = requests.get(url, timeout=10) - - # Step 3: Use regex to search for HTML tags in the content - if re.search(r"", response.text, re.IGNORECASE): - return True - else: - return False - except requests.RequestException as e: - print(f"Error checking URL: {e}") - return False - - -def find_by_relative_search(input_text, is_support_opposite = False): - checked_urls = set() - searched_candidates = [] - - # Get most frequent words - top_words = get_most_frequent_words(input_text) - - # Find identities - # entities = extract_entities(input_text) - - # Make a search text based on the most frequent words and entities - # searched_candidates.append(" ".join(entities[:16]) + " " + " ".join(top_words[:16])) - - # Find phrases - searched_candidates = searched_candidates + get_candidate_phrase_for_relative_search(input_text) - - for candidate in searched_candidates: - search_results = google_search(candidate, GOOGLE_API_KEY, SEARCH_ENGINE_ID, is_exactTerms = False) - urls = [item['link'] for item in search_results.get("items", [])] - - for url in urls[:5]: - if url in checked_urls: # already checked - continue - checked_urls.add(url) - size = get_url_size(url) - if size != None and size <= MAX_URL_SIZE: - if check_if_html(url): - paraphrase_threshold = PARAPHRASE_THRESHOLD - if is_support_opposite: - paraphrase_threshold = PARAPHRASE_THRESHOLD_FOR_OPPOSITE - is_paraphrase, data = check_paraphrase(input_text, url, paraphrase_threshold = paraphrase_threshold) - if is_paraphrase: - return is_paraphrase, url, data - # else: - # print(f"ignore {url} due to size = {size}") - return False, None, [] - - -PARAPHASE_MODEL = None - -def split_to_sentences(input_text): - """ - Chia input text dựa trên dấu xuống dòng - sentence tokenize từng paragraph - """ - paragraphs = input_text.split("\n") - result = [] - for paragraph in paragraphs: - paragraph = paragraph.strip() - if paragraph != "": - sentences = sent_tokenize(paragraph) - result.extend(sentences) - return result - -def longest_common_subsequence(arr1, arr2): - n = len(arr1) - m = len(arr2) - - # Tạo bảng dp kích thước (n+1) x (m+1) - dp = [[0] * (m + 1) for _ in range(n + 1)] - max_length = 0 # Biến lưu trữ chiều dài lớn nhất của dãy con chung - - for i in range(1, n + 1): - for j in range(1, m + 1): - # Nếu phần tử trùng nhau - if arr1[i - 1] == arr2[j - 1]: - dp[i][j] = dp[i - 1][j - 1] + 1 - max_length = max(max_length, dp[i][j]) - else: - dp[i][j] = 0 # Đặt về 0 vì dãy con phải liên tục - - return max_length - - -def check_individual_sentence(input_sentence, source_sentence, min_same_sentence_len, min_phrase_sentence_len, verbose = False): - input_sent = input_sentence.strip() - source_sent = source_sentence.strip() - input_words = input_sent.split(" ") - source_words = source_sent.split(" ") - result = False - if input_sent == source_sent and len(input_words) >= min_same_sentence_len: - result = True - else: - max_overlap_len = longest_common_subsequence(input_words, source_words) - if max_overlap_len >= min_phrase_sentence_len: - result = True - - if verbose: - if result: - max_overlap_len = longest_common_subsequence(input_words, source_words) - return result - -def download_file(url, output_dir="downloads"): - """ - Downloads a file from the given URL and saves it locally. - """ - response = requests.get(url, stream=True) - if response.status_code == 200: - os.makedirs(output_dir, exist_ok=True) - file_name = url.split("/")[-1] - file_path = os.path.join(output_dir, file_name) - with open(file_path, "wb") as file: - file.write(response.content) - return file_path - else: - print(f"Failed to download {url}: {response.status_code}") - return None - -def extract_text_from_pdf(file_path): - """ - Extracts text from a PDF file. - """ - reader = PdfReader(file_path) - text = "" - for page in reader.pages: - text += page.extract_text() - return text - -def extract_text_from_docx(file_path): - """ - Extracts text from a DOCX file. - """ - doc = Document(file_path) - text = "" - for paragraph in doc.paragraphs: - text += paragraph.text + "\n" - return text - - -def extract_text_from_html(url): - """ - Extracts text from an HTML page. - """ - response = requests.get(url) - soup = BeautifulSoup(response.text, "html.parser") - return soup.get_text(separator="\n") - -def extract_text(url): - """ - Determines the file type and extracts text accordingly. - """ - - try: - file_extension = url.split('.')[-1].lower() - if file_extension in ["html", "htm"]: - return extract_text_from_html(url) - elif file_extension == "pdf": - file_path = download_file(url) - return extract_text_from_pdf(file_path) if file_path else None - elif file_extension in ["doc", "docx"]: - file_path = download_file(url) - return extract_text_from_docx(file_path) if file_path else None - else: - print(f"Unsupported file type: {file_extension}") - return extract_text_from_html(url) - except: - return "" - - -def check_paraphrase( - input_text, - url, - paraphrase_threshold = PARAPHRASE_THRESHOLD, - min_ratio = MIN_RATIO_PARAPHASE_NUM, - min_same_sentence_len = MIN_SAME_SENTENCE_LEN, - min_phrase_sentence_len = MIN_PHRASE_SENTENCE_LEN, - verbose = False): - """ - Check input_text and url có paraphrase or not: - + input - - input_text: - - url: - - paraphrase_threshold: cosine similarity tối thiểu để kết luận là paraphrase - - min_ratio: ratio tối thiểu của số lượng câu trong input_text tìm được paraphrase (làm tròn lên) - + output - - True/False => paraphrase or not - - a list element. each element - . input sentence - . matchted sentence (from source) - . Similarity - . True/False stastify the threshold - """ - is_paraphrase_text = False - - if input_text == None: - return is_paraphrase_text, [] - input_sentences = split_to_sentences(input_text) - page_text = extract_text(url) - - if page_text == None: - return is_paraphrase_text, [] - page_sentences = split_to_sentences(page_text) - if len(input_sentences) == 0 or len(page_sentences) == 0: - return is_paraphrase_text, [] - global PARAPHASE_MODEL - if PARAPHASE_MODEL == None: - PARAPHASE_MODEL = SentenceTransformer('paraphrase-MiniLM-L6-v2') - PARAPHASE_MODEL.to(DEVICE) - total_sentence = len(input_sentences) - min_matching = int(math.ceil(total_sentence * min_ratio)) - - # Encode sentences into embeddings - embeddings1 = PARAPHASE_MODEL.encode(input_sentences, convert_to_tensor=True, device=DEVICE) - embeddings2 = PARAPHASE_MODEL.encode(page_sentences, convert_to_tensor=True, device=DEVICE) - - # Compute cosine similarity between each pair of sentences - similarity_matrix = util.cos_sim(embeddings1, embeddings2).cpu().numpy() - - # Align sentences - alignment = [] - count = 0 - - for i, sentence1 in enumerate(input_sentences): - max_sim_index = np.argmax(similarity_matrix[i]) - max_similarity = similarity_matrix[i][max_sim_index] - if max_similarity > paraphrase_threshold: # Threshold for paraphrase alignment - is_paraphrase_sentence = True - count += 1 - else: - is_paraphrase_sentence = False - - - item = [sentence1, page_sentences[max_sim_index], max_similarity, is_paraphrase_sentence] - if is_paraphrase_text==False and check_individual_sentence(sentence1, page_sentences[max_sim_index], min_same_sentence_len, min_phrase_sentence_len): - is_paraphrase_text = True - if verbose: - print(f"sentence1 = {sentence1}") - print(f"page_sentences[max_sim_index] = {page_sentences[max_sim_index]}") - alignment.append(item) - if count >= min_matching: - is_paraphrase_text = True - - if verbose: - print(f"min_matching = {min_matching}") - print(f"len(input_sentences) = {len(input_sentences)}") - print(f"count = {count}") - print(f"is_paraphrase_text = {is_paraphrase_text}") - for item in alignment: - print(item) - return is_paraphrase_text, alignment - - -def get_url_size_by_head(url): - try: - response = requests.head(url, allow_redirects=True) - if 'Content-Length' in response.headers: - size = int(response.headers['Content-Length']) - return size - else: - print("Content-Length header is not available.") - return None - except requests.RequestException as e: - print(f"Error: {e}") - return None - -def get_url_size(url): - size = get_url_size_by_head(url) - return size - -def get_top_words_without_stop_words(input_text, number_word = 15): - words = word_tokenize(input_text) - - stop_words = set(stopwords.words('english')) - filtered_words = [word for word in words if word.isalnum() and word.lower() not in stop_words] - word_frequencies = Counter(filtered_words) - top_words = word_frequencies.most_common(number_word) - - return top_words - -if __name__ == "__main__": - pass diff --git a/src/texts/Search_Text/_text_detection_share.py b/src/texts/Search_Text/_text_detection_share.py deleted file mode 100644 index 5d1abe9bc2261d654d72ef1b882561b143912717..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/_text_detection_share.py +++ /dev/null @@ -1,100 +0,0 @@ -from transformers import pipeline -from _google_search_engine_testing_share import find_by_relative_search -import math - -PROOFREAD_FILE = "data/1_proofread/xsum/gpt-4o-mini_with_best_similarity.csv" -WORD_FREQUENCY = None - -DEFAULT_MODEL = "Hello-SimpleAI/chatgpt-detector-roberta" -""" -data/MAGE/xsum_human.csv = {'HUMAN': 64, 'MACHINE': 36} correction = 20 => 84% -data/MAGE/xsum_machine_topical_gpt-3.5-trubo.csv = {'HUMAN': 3, 'MACHINE': 97} => correction = 3 => 94% - original acc = (64+97)/ 200 = 80.5% - improve = (84 + 94) / 200 = 89% - different = 8.5% - -https://huggingface.co/datasets/RealTimeData/bbc_news_alltime = {'HUMAN': 82, 'MACHINE': 18} => corrected 16 => 98% - -""" - -MODEL_HUMAN_MATCHING = dict() -MODEL_HUMAN_MATCHING[DEFAULT_MODEL] = "Human" - -HUMAN = "HUMAN" -MACHINE = "MACHINE" - -UNKNOWN = "UNKNOWN" -PARAPHASE = "PARAPHASE" -NON_PARAPHASE = "NON_PARAPHASE" - - -def detect_by_huggingface_model(input_text, model = DEFAULT_MODEL, max_length=512): - """ - trả về kết quả là "HUMAN" hay "MACHINE" và confidence score (int) - """ - pipe = pipeline("text-classification", model=model,tokenizer=model, max_length=512, truncation=True, device_map="auto") - result = pipe(input_text)[0] - confidence_score = result['score'] - if result['label'] == MODEL_HUMAN_MATCHING[model]: - return HUMAN, confidence_score - else: - return MACHINE, confidence_score - -def check_human(data, min_ratio = 0.7): - """ - input: - - data have item: - + input sentence - + source sentence - + similarity - + True/False : paraphrase or not - output: - is human (True/False) - """ - total_sentence = len(data) - min_matching = int(math.ceil(total_sentence * min_ratio)) - count = 0 - for input_sentence, source_sentence, similiarity, is_paraprhase in data: - if input_sentence in source_sentence: - count += 1 - if count >= min_matching: - return True - else: - return False - -def abstract_detect_generated_text(input_text): - """ - Assists to detect the source of text using the search engine - Output - - prediction by search engine (HUMAN/MACHINE/UNKNOWN) - - Prediction by SOTA (HUMAN/MACHINE) - - SOTA confidence (float) - - url to website (None if UNKNOWN) - - pair of sentences. Each item have ([] if empty) - - input sentence - - source sentence best matching in url - - matching result between input /source sentence (PARAPHASE/NON_PARAPHASE) - """ - is_support_opposite = False - is_paraphrase, found_url, data = find_by_relative_search(input_text, is_support_opposite) - sentence_pairs = [] - SOTA_prediction, SOTA_confidence = detect_by_huggingface_model(input_text) - if not is_paraphrase: - search_engine_prediction = UNKNOWN - else: - if check_human(data): - search_engine_prediction = HUMAN - else: - search_engine_prediction = MACHINE - for input_sentence, source_sentence, similiarity, is_paraphrase in data: - if is_paraphrase: - check_paraphrase = PARAPHASE - else: - check_paraphrase = NON_PARAPHASE - sentence_pairs.append([input_sentence, source_sentence, check_paraphrase]) - - return search_engine_prediction, SOTA_prediction, SOTA_confidence, found_url, sentence_pairs - -if __name__ == "__main__": - pass - diff --git a/src/texts/Search_Text/chatgpt_detector_roberta.py b/src/texts/Search_Text/chatgpt_detector_roberta.py deleted file mode 100644 index 14f6ebb3dc095bda0d314400cc48732c4bd317db..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/chatgpt_detector_roberta.py +++ /dev/null @@ -1,119 +0,0 @@ -import math - -from _google_search_engine_testing_share import find_by_relative_search -from transformers import pipeline - -# TODO: move to a config file -# Constants should be UPPER_SNAKE_CASE -PROOFREAD_FILE = "data/1_proofread/xsum/gpt-4o-mini_with_best_similarity.csv" -WORD_FREQUENCY = None - -DEFAULT_MODEL = "Hello-SimpleAI/chatgpt-detector-roberta" - -MODEL_HUMAN_LABEL = {DEFAULT_MODEL: "Human"} - -HUMAN = "HUMAN" -MACHINE = "MACHINE" -UNKNOWN = "UNKNOWN" -PARAPHRASE = "PARAPHRASE" -NON_PARAPHRASE = "NON_PARAPHRASE" - - -def detect_ai_content( - input_text: str, - model: str = DEFAULT_MODEL, - max_length: int = 512, -) -> tuple: - """ - Detects if text is human or machine generated. - - Returns: - tuple: (label, confidence_score) - where label is HUMAN or MACHINE. - """ - try: - pipe = pipeline( - "text-classification", - model=model, - tokenizer=model, - max_length=max_length, - truncation=True, - device_map="auto", # good for GPU usage - ) - result = pipe(input_text)[0] - confidence_score = result["score"] - if result["label"] == MODEL_HUMAN_LABEL[model]: - label = HUMAN - else: - label = MACHINE - return label, confidence_score - except Exception as e: # Add exception handling - print(f"Error in Roberta model inference: {e}") - return UNKNOWN, 0.0 # Return UNKNOWN and 0.0 confidence if error - - -def check_human(data, min_ratio=0.7): - """ - Checks if a sufficient number of input sentences are found within - source sentences. - - Returns: - bool: True if the condition is met, False otherwise. - """ - if not data: # Handle empty data case - return False - min_matching = math.ceil(len(data) * min_ratio) - - count = 0 - - #for input_sentence, source_sentence, similiarity, is_paraprhase in data: - for sentence in data: - if sentence["similarity"] >= 0.99: - count += 1 - print(f"\tmatching_sentence_count : {count}, min_matching: {min_matching}") - if count >= min_matching: - return True - return False - - -def abstract_detect_generated_text(input_text): - """ - Abstracts the process of detecting generated text using search - and a classification model. - - Returns: - tuple: ( - search_engine_prediction, - SOTA_prediction, - SOTA_confidence, - found_url, - sentence_pairs, - ) - """ - - is_paraphrase, found_url, data = find_by_relative_search( - input_text, - is_support_opposite=False, - ) # Explicitly set the keyword argument - SOTA_prediction, SOTA_confidence = detect_ai_content(input_text) - - if not is_paraphrase: - search_engine_prediction = UNKNOWN - else: - search_engine_prediction = HUMAN if check_human(data) else MACHINE - - sentence_pairs = [] - if data: # Check if data is not empty to avoid error when iterating - for input_sentence, source_sentence, _, is_paraphrase in data: - check_paraphrase = PARAPHRASE if is_paraphrase else NON_PARAPHRASE - sentence_pairs.append( - [input_sentence, source_sentence, check_paraphrase], - ) - - return ( - search_engine_prediction, - SOTA_prediction, - SOTA_confidence, - found_url, - sentence_pairs, - ) diff --git a/src/texts/Search_Text/comparison.py b/src/texts/Search_Text/comparison.py deleted file mode 100644 index f56464798117efe2c21a0235570588152e8405fb..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/comparison.py +++ /dev/null @@ -1,217 +0,0 @@ -import pandas as pd -import re -import csv -from collections import Counter -from difflib import Differ -import nltk -from nltk.corpus import stopwords -nltk.download('stopwords') - - -def remove_stop_words(word_list): - """ - Removes stop words from a list of single words. - - Args: - word_list: A list of single words. - - Returns: - A new list containing only the words that are not stop words. - """ - - stop_words = set(stopwords.words('english')) # Get English stop words - - # Define characters to remove - chars_to_remove = r'[^a-zA-Z0-9]' # Matches any character that is not a letter or digit - - cleaned_words = [] - for word in word_list: - # Remove punctuation and special characters - word = re.sub(chars_to_remove, '', word) - - # Check for single digits and single letters - if len(word) > 1 and not word.isdigit(): - # Check if the word is not a stop word - if word.lower() not in stop_words: - cleaned_words.append(word) - - return cleaned_words - - -def write_word_counts_to_csv(data): - """Writes word counts to a CSV file from a dictionary. - - Args: - data_dict: A dictionary containing the word count data. - filename: The name of the output CSV file. - """ - - with open('data/results/[res]added_word_counts.csv', 'w', encoding='utf-8', newline='') as csvfile: - fieldnames = ['Word', 'Count'] - writer = csv.writer(csvfile) - writer.writerow(fieldnames) - - for word, count in data['added_word_counts']: - writer.writerow([word, count]) - - with open('data/results/[res]removed_word_counts.csv', 'w', encoding='utf-8', newline='') as csvfile: - fieldnames = ['Word', 'Count'] - writer = csv.writer(csvfile) - writer.writerow(fieldnames) - - for word, count in data['removed_word_counts']: - writer.writerow([word, count]) - - # with open('data/results/[res]unchanged_words.csv', 'w', encoding='utf-8', newline='') as csvfile: - # fieldnames = ['Count', 'Phrase'] - # writer = csv.writer(csvfile) - # writer.writerow(fieldnames) # Write the header - # for phrase, count in data['unchanged_words']: - # writer.writerow([count, phrase]) - - -def preprocess_text(text): - """ - Preprocesses a string by removing punctuation, numbers, and whitespace. - - Args: - text: The string to preprocess. - - Returns: - The preprocessed string. - """ - - # Lower case - text = text.lower() - - # Split text into words while keeping commas and dots within numbers - delimiters = r"(?= 4: - unchanged_phrase = " ".join(substring.split()) - unchanged_phrases.append((unchanged_phrase, count)) - substring = "" - count = 0 - continue - substring += " " + word - count += 1 - - return removed_ngrams, added_ngrams, unchanged_phrases - - -if __name__ == "__main__": - res = compare_strings_from_csv("data/ChatGPT_Nous_Hermes_2_Yi_34B_openchat_3_5_1210_with_best_similarity.csv") - write_word_counts_to_csv(res) - - #remove_stop_words(["the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"]) \ No newline at end of file diff --git a/src/texts/Search_Text/evaluation.py b/src/texts/Search_Text/evaluation.py deleted file mode 100644 index adc77c7741304c800a1adebf540a5adf645190a1..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/evaluation.py +++ /dev/null @@ -1,138 +0,0 @@ -import csv -import time - -import pandas as pd -from chatgpt_detector_roberta import ( - check_human, - detect_ai_content, -) -from search_text import detect_by_relative_search - -HUMAN = "HUMAN" -MACHINE = "MACHINE" - - -def read_csv_column(file_path, column_name, data_size=100): - """ - Reads a CSV file and extracts data from the specified column. - - Args: - filename: Path to the CSV file. - column_name: Name of the column to extract data from. - - Returns: - A list containing the data from the specified column. - """ - - try: - df = pd.read_csv(file_path) - column_data = df[column_name].tolist() - return column_data[:data_size] - except FileNotFoundError: - print(f"Error: File '{file_path}' not found.") - return [] - except KeyError: - print(f"Error: Column '{column_name}' not found in the CSV file.") - return [] - - -def evaluation(texts): - results = [] - index = 0 - for text in texts: - if index <= 82: - print(f"index = {index}") - index += 1 - continue - - # Classify by SOTA model - # SOTA_prediction, SOTA_confidence = detect_by_huggingface_model(text) - SOTA_prediction, SOTA_confidence = detect_ai_content(text) - - # Classify by search engine - # is_paraphrased, _, data = find_by_relative_search(text) - is_paraphrased, _, data = detect_by_relative_search(text) - if not is_paraphrased: - search_engine_prediction = "UNKNOWN" - else: - if check_human(data): - search_engine_prediction = HUMAN - else: - search_engine_prediction = MACHINE - print( - f"RESULTS:\t{SOTA_prediction}\t{search_engine_prediction}" - ) - results.append( - (index, SOTA_prediction, SOTA_confidence, search_engine_prediction) - ) - - with open("eva_bbc_test.csv", "a", newline="") as csvfile: - #with open("eva_MAGE_test.csv", "a", newline="") as csvfile: - writer = csv.writer(csvfile) - writer.writerow( - [index, SOTA_prediction, SOTA_confidence, search_engine_prediction] - ) - index += 1 - time.sleep(1) # avoid 100? queries per minute limit - - # Define the column names - # columns = [ - # "index", - # "SOTA_prediction", - # "SOTA_confidence", - # "search_engine_prediction", - # ] - - # # Create the DataFrame - # df = pd.DataFrame(results, columns=columns) - - # # Statistics - # search_engine_acc = df["search_engine_prediction"].value_counts()[ - # "HUMAN" - # ] / len(df) - # SOTA_acc = df["SOTA_prediction"].value_counts()["HUMAN"] / len(df) - - # # Filter the DataFrame based on the given conditions - # filtered_df = df[ - # (df["SOTA_prediction"] == "MACHINE") - # & (df["search_engine_prediction"] == "HUMAN") - # ] - - # print(f"Total data: {len(df)}") - # print(f"SOTA accuracy: {SOTA_acc}") - # print(f"Search engine accuracy: {search_engine_acc}") - # print(f"Correction sample: {len(filtered_df)}") - - -def extract_machine_data(file_path): - df = pd.read_csv(file_path) - machine_data = df[df["src"] == "xsum_machine_topical_gpt-3.5-trubo"] - - # write to file - machine_data.to_csv("machine_data.csv", index=False) - -def extract_human_data(file_path): - df = pd.read_csv(file_path) - machine_data = df[df["src"] == "xsum_human"] - - # write to file - machine_data.to_csv("machine_data.csv", index=False) - - -if __name__ == "__main__": - # extract_machine_data('data/test_data/test.csv') - - # BBC - file_path = "data/test_data/test_100_bbc.csv" - column_name = "content" - - # MAGE - # file_path = "data/test_data/test_100_MAGE.csv" - # column_name = "text" - - contents = read_csv_column( - file_path=file_path, - column_name=column_name, - data_size=100, - ) - evaluation(contents) \ No newline at end of file diff --git a/src/texts/Search_Text/fake_text_generation_share.py b/src/texts/Search_Text/fake_text_generation_share.py deleted file mode 100644 index e9ec69d7bc7484cd53aa9b372ae441f31cf6a2ca..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/fake_text_generation_share.py +++ /dev/null @@ -1,53 +0,0 @@ -from difflib import SequenceMatcher - - - -def highlight_overlap_by_word_to_list(text1, text2): - """ - trả về: - - list of words in text1 - - list of words in text2 - - list of index of hight words in text 1 - - list of index of hight words in text 2 - """ - # Tách chuỗi thành các từ (word) dựa vào khoảng trắng - words1 = text1.split() - words2 = text2.split() - - index1 = [] - index2 = [] - - # Sử dụng SequenceMatcher để tìm các đoạn trùng lặp giữa danh sách các từ - matcher = SequenceMatcher(None, words1, words2) - - highlighted_text1 = [] - highlighted_text2 = [] - - # Theo dõi vị trí hiện tại trong words1 và words2 - current_pos1 = 0 - current_pos2 = 0 - - # Lặp qua các đoạn so khớp - for match in matcher.get_matching_blocks(): - start1, start2, length = match - - # Thêm các từ không trùng lặp vào (giữ nguyên) - highlighted_text1.extend(words1[current_pos1:start1]) - highlighted_text2.extend(words2[current_pos2:start2]) - - if length > 0: - for i in range(start1, start1 + length): - index1.append(i) - for i in range(start2, start2 + length): - index2.append(i) - - - # Cập nhật vị trí hiện tại - current_pos1 = start1 + length - current_pos2 = start2 + length - - return words1, words2, index1, index2 - - -if __name__ == "__main__": - pass diff --git a/src/texts/Search_Text/identity.py b/src/texts/Search_Text/identity.py deleted file mode 100644 index e05f5d34beaf8484600aff4d3bb1502915aff3ab..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/identity.py +++ /dev/null @@ -1,63 +0,0 @@ -from transformers import pipeline - -ner_pipeline = pipeline("ner") - -def extract_entities(text): - output = ner_pipeline(text) - words = extract_words(output) - words = combine_subwords(words) - - # extract word in each entity and assign to a list of entities, connect words if there is no space between them - entities = [] - for entity in words: - if entity not in entities: - entities.append(entity) - - return entities - - -def extract_words(entities): - """ - Extracts the words from a list of entities. - - Args: - entities: A list of entities. - - Returns: - A list of words extracted from the entities. - """ - words = [] - for entity in entities: - words.append(entity["word"]) - return words - - -def combine_subwords(word_list): - """ - Combines subwords (indicated by "##") with the preceding word in a list. - - Args: - word_list: A list of words, where subwords are prefixed with "##". - - Returns: - A new list with subwords combined with their preceding words. - """ - result = [] - i = 0 - while i < len(word_list): - if word_list[i].startswith("##"): - result[-1] += word_list[i][2:] # Remove "##" and append to the previous word - elif i < len(word_list) - 2 and word_list[i + 1] == "-": # Combine hyphenated words - result.append(word_list[i] + word_list[i + 1] + word_list[i + 2]) - i += 2 # Skip the next two words - else: - result.append(word_list[i]) - i += 1 - return result - -if __name__ == "__main__": - text = "The Saudi authorities, I am told, are currently working flat out" \ - "to collate everything they have on the Magdeburg market suspect," \ - "Taleb al-Abdulmohsen, and to share it with Germany's ongoing" \ - "investigation" - print(extract_entities(text)) \ No newline at end of file diff --git a/src/texts/Search_Text/search_text.py b/src/texts/Search_Text/search_text.py deleted file mode 100644 index 4b0c426ddc8936e98707598961293b1e2deb4142..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/search_text.py +++ /dev/null @@ -1,791 +0,0 @@ -import warnings - -from bs4 import BeautifulSoup - -from identity import extract_entities -warnings.simplefilter(action='ignore', category=FutureWarning) - -import time -import numpy as np -import pandas as pd -from sklearn.feature_extraction.text import TfidfVectorizer -import re -from collections import Counter -import string -import nltk -import torch -from nltk.corpus import stopwords -from nltk.tokenize import sent_tokenize, word_tokenize -from nltk.util import ngrams -from sentence_transformers import SentenceTransformer, util -import math - -from dotenv import load_dotenv -from difflib import SequenceMatcher -import os -import requests -import csv -from newspaper import article, ArticleException, ArticleBinaryDataException - - -# Google Cloud Console -load_dotenv() -GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") -SEARCH_ENGINE_ID = os.getenv("SEARCH_ENGINE_ID") - -# Download necessary NLTK data files -nltk.download('punkt', quiet=True) -nltk.download('punkt_tab', quiet=True) -nltk.download('stopwords', quiet=True) - -# load the model -DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -PARAPHASE_MODEL = SentenceTransformer('paraphrase-MiniLM-L6-v2') -PARAPHASE_MODEL.to(DEVICE) - -BATCH_SIZE = 8 -MAX_URL_SIZE = 2000000 # ~2MB - -PARAPHRASE_THRESHOLD = 0.8 -PARAPHRASE_THRESHOLD_FOR_OPPOSITE = 0.7 -MIN_SAME_SENTENCE_LEN = 6 -MIN_PHRASE_SENTENCE_LEN = 10 -MIN_RATIO_PARAPHRASE_NUM = 0.7 -MAX_CHAR_SIZE = 30000 - - -def clean_text(text): - """Doc cleaning""" - punctuations = r"""!"#$%&'()*+-/:;<=>?@[\]^_`{|}~""" # not include , and . due to number - # Lowering text - text = text.lower() - - # Removing punctuation - text = "".join([c for c in text if c not in punctuations]) - - # Removing whitespace and newlines - text = re.sub(r'\s+',' ',text) - - text.replace("£", " * ") - - words = text.split() - text = ' '.join(words[:18]) # Join the first 18 words back into a string - - return text - -def remove_punctuation(text): - """Remove punctuation from a given text.""" - punctuation_without_dot = string.punctuation.replace(".", "") - translator = str.maketrans('', '', punctuation_without_dot) - return text.translate(translator) - -def get_keywords(text, num_keywords=5): - """Return top k keywords from a doc using TF-IDF method""" - - # Create a TF-IDF Vectorizer - vectorizer = TfidfVectorizer(stop_words='english') - - # Fit and transform the text - tfidf_matrix = vectorizer.fit_transform([text]) - - # Get feature names (words) - feature_names = vectorizer.get_feature_names_out() - - # Get TF-IDF scores - tfidf_scores = tfidf_matrix.toarray()[0] - - # Sort words by TF-IDF score - word_scores = list(zip(feature_names, tfidf_scores)) - word_scores.sort(key=lambda x: x[1], reverse=True) - - # Return top keywords - return [word for word, score in word_scores[:num_keywords]] - -""" -# Example usage -text = "Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to natural intelligence displayed by animals including humans. Leading AI textbooks define the field as the study of "intelligent agents": any system that perceives its environment and takes actions that maximize its chance of achieving its goals. Some popular accounts use the term "artificial intelligence" to describe machines that mimic "cognitive" functions that humans associate with the human mind, such as "learning" and "problem solving", however this definition is rejected by major AI researchers." -print(f"\n# Input text:\n'{text}'") -print("\n----------------------\n") - -keywords = get_keywords(text) -print("# Top keywords:", keywords) -print("\n----------------------\n") -""" - -def get_important_sentences(paragraph: str, keywords: list[str], num_sentences: int = 3) -> list[str]: - """ - Selects important sentences from a given paragraph based on a list of keywords. - - Args: - paragraph (str): The input paragraph. - keywords (list[str]): List of important keywords. - num_sentences (int): Number of sentences to return (default is 3). - - Returns: - list: A list of important sentences. - """ - # Clean and split the paragraph into sentences - sentences = [s.strip() for s in re.split(r'(?<=[.!?])\s+', paragraph) if s.strip()] - - # Calculate the importance score for each sentence - sentence_scores = [] - for sentence in sentences: - processed_sentence = clean_text(sentence) - score = 0 - words = processed_sentence.lower().split() - word_count = Counter(words) - - for keyword in keywords: - if keyword.lower() in word_count: - score += word_count[keyword.lower()] - - sentence_scores.append((sentence, score)) - - # Sort sentences by their scores in descending order - sentence_scores.sort(key=lambda x: x[1], reverse=True) - - # Return the top N sentences - return [sentence for sentence, score in sentence_scores[:num_sentences]] - -"""# Example usage -keywords = get_keywords(paragraph) -important_sentences = get_important_sentences(paragraph, keywords) - -print("# Important sentences:") -for i, sentence in enumerate(important_sentences, 1): - print(f"{i}. {sentence}") -print("\n----------------------\n") -""" - -def extract_important_phrases(paragraph: str, keywords: list[str], phrase_length: int = 5) -> list[str]: - """ - Extracts important phrases from a given paragraph based on a list of keywords. - Phrase length is auto-determined, and overlapped parts are less than 20%. - - Args: - paragraph (str): The input paragraph. - keywords (list[str]): List of important keywords. - phrase_length (int): The length of phrases to extract (default is 5 words). - - Returns: - list: A list of important phrases. - """ - # Tokenize the paragraph into words - words = word_tokenize(paragraph.lower()) - - # Determine phrase length (between 3 and 7 words) - phrase_length = min(max(len(words) // 10, 5), 7) - - # Generate n-grams (phrases) from the paragraph - phrases = list(ngrams(words, phrase_length)) - - important_phrases = [] - used_indices = set() - - for i, phrase in enumerate(phrases): - # Check if the phrase contains any keyword - if any(keyword.lower() in phrase for keyword in keywords): - # Check overlap with previously selected phrases - if not any(abs(i - j) < phrase_length * 0.8 for j in used_indices): - important_phrases.append(clean_text(" ".join(phrase))) - used_indices.add(i) - - return important_phrases - -"""# Example usage -keywords = get_keywords(paragraph) -important_phrases = extract_important_phrases(paragraph, keywords) - -print("# Important phrases:") -for i, phrase in enumerate(important_phrases[:5], 1): # Print top 5 phrases - print(f"{i}. {phrase}")""" - -def search_by_google( - query, - num_results=10, - is_exact_terms = False - ) -> dict: - """ - Searches the Google Custom Search Engine for the given query. - - Args: - query: The search query. - is_exact_terms: Whether to use exact terms search (True) or regular search (False). - num_results: The number of results to return (default: 10). - - Returns: - A dictionary containing the search results or None if there was an error. - """ - - start_date = "20000101" - end_date = "20210101" - - url = "https://www.googleapis.com/customsearch/v1" - params = { - "key": GOOGLE_API_KEY, - "cx": SEARCH_ENGINE_ID, - "num": num_results, - } - if is_exact_terms: - params["exactTerms"] = query - else: - params["q"] = query.replace('"', "") - - response = requests.get(url, params=params) - if response.status_code == 200: - return response.json() - else: - print(f"Error: {response.status_code}, {response.text}") - return None - - -def display_Google_results(results): - for result in results: - print(f"Title: {result['title']}") - print(f"Link: {result['link']}") - print(f"Snippet: {result['snippet']}") - print(" ------- ") - - -def detect_by_relative_search(input_text, is_support_opposite = False): - checked_urls = set() - searched_phrases = generate_search_phrases(input_text) - - for candidate in searched_phrases: - search_results = search_by_google(candidate) - urls = [item['link'] for item in search_results.get("items", [])] - - for url in urls[:3]: - if url in checked_urls: # already checked - continue - checked_urls.add(url) - print(f"\n\tURL: {url}") - size = get_url_size(url) - if size != None and size <= MAX_URL_SIZE: - page_text = extract_text(url) - if page_text is None or len(page_text) > MAX_CHAR_SIZE: - print(f"\t\t↑↑↑ More than {MAX_CHAR_SIZE} characters") - continue - is_paraphrase, data = check_paraphrase(input_text, page_text) - if is_paraphrase: - return is_paraphrase, url, data - return False, None, [] - -def get_url_size(url): - """ - Retrieves the size of a URL's content using a HEAD request. - - Args: - url: The URL to check. - - Returns: - The size of the content in bytes, or None if the size cannot be determined - (e.g., due to network errors or missing Content-Length header). - """ - try: - response = requests.head(url, allow_redirects=True, timeout=5) # Add timeout - response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx) - - content_length = response.headers.get('Content-Length') - if content_length is not None: - return int(content_length) - else: - print(f"\t\t↑↑↑ Content-Length header not found") - return None - - except requests.exceptions.RequestException as e: - print(f"\t\t↑↑↑ Error getting URL size: {e}") - return None - -def get_most_frequent_words(input_text, number_word=32): - """ - Gets the top words from the input text, excluding stop words and punctuation. - - Args: - input_text: The input text as a string. - number_word: The number of top words to return. - - Returns: - A list of tuples, where each tuple contains a word and its frequency. - Returns an empty list if input is not a string or is empty. - """ - if not isinstance(input_text, str) or not input_text: - return [] - - words = word_tokenize(input_text.lower()) # Tokenize and lowercase - - stop_words = set(stopwords.words('english')) - punctuation = set(string.punctuation) # get all punctuation - filtered_words = [ - word for word in words - if word.isalnum() and word not in stop_words and word not in punctuation - ] - word_frequencies = Counter(filtered_words) - top_words = word_frequencies.most_common(number_word) - - for top_word in top_words: - words.append(top_word[0]) - - if len(words) > 32: - search_phrase = " ".join(words[:32]) - else: - search_phrase = " ".join(words[:number_word]) - - return search_phrase - -def get_chunk(input_text, chunk_length=32, num_chunk=3): - """ - Splits the input text into chunks of a specified length. - - Args: - input_text: The input text as a string. - num_chunk: The maximum number of chunks to create. - chunk_length: The desired length of each chunk (in words). - - Returns: - A list of string chunks. - Returns an empty list if input is invalid. - """ - if not isinstance(input_text, str): - return [] - - chunks = [] - input_words = input_text.split() # Split by any whitespace - - for i in range(num_chunk): - start_index = i * chunk_length - end_index = (i + 1) * chunk_length - chunk = " ".join(input_words[start_index:end_index]) - if chunk: # Only append non-empty chunks - chunks.append(chunk) - - return chunks - -def generate_search_phrases(input_text): - """ - Generates different types of phrases for search purposes. - - Args: - input_text: The input text. - - Returns: - A list containing: - - A list of most frequent words. - - The original input text. - - A list of text chunks. - """ - if not isinstance(input_text, str): - return [] - - search_phrases = [] - - # Method 1: Get most frequent words - search_phrases.append(get_most_frequent_words(input_text)) - - # Method 2: Get the whole text - search_phrases.append(input_text) - - # Method 3: Split text by chunks - search_phrases.extend(get_chunk(input_text)) - - # Method 4: Get most identities and key words - entities = extract_entities(input_text) - keywords = get_keywords(input_text, 16) - search_phrase = " ".join(entities) + " " + " ".join(keywords) - search_phrases.append(search_phrase) - - return search_phrases - -def split_into_sentences(input_text): - """ - Splits input text into sentences by newlines. - - Args: - input_text: The input text as a string. - - Returns: - A list of sentences. Returns an empty list if input is not valid. - """ - if not isinstance(input_text, str): - return [] - - paragraphs = input_text.splitlines() - sentences = [] - for paragraph in paragraphs: - paragraph = paragraph.strip() - if paragraph: - sentences.extend(sent_tokenize(paragraph)) - return sentences - - -def longest_common_subsequence(arr1, arr2): - """ - Finds the length of the longest common subsequence (contiguous) between - two arrays. - - Args: - arr1: The first array. - arr2: The second array. - - Returns: - The length of the longest common subsequence. - Returns 0 if either input is invalid. - """ - - if not isinstance(arr1, list) or not isinstance(arr2, list): - return 0 - - n = len(arr1) - m = len(arr2) - - if n == 0 or m == 0: #handle empty list - return 0 - - # Create table dp with size (n+1) x (m+1) - dp = [[0] * (m + 1) for _ in range(n + 1)] - max_length = 0 - - for i in range(1, n + 1): - for j in range(1, m + 1): - if arr1[i - 1] == arr2[j - 1]: - dp[i][j] = dp[i - 1][j - 1] + 1 - max_length = max(max_length, dp[i][j]) - else: - dp[i][j] = 0 # set 0 since the array must be consecutive - - return max_length - - -def check_sentence(input_sentence, source_sentence, min_same_sentence_len, - min_phrase_sentence_len, verbose=False): - """ - Checks if two sentences are similar based on exact match or - longest common subsequence. - - Args: - input_sentence: The input sentence. - source_sentence: The source sentence. - min_same_sentence_len: Minimum length for exact sentence match. - min_phrase_sentence_len: Minimum length for common subsequence match. - verbose: If True, print debug information. - - Returns: - True if the sentences are considered similar, False otherwise. - Returns False if input is not valid. - """ - - if not isinstance(input_sentence, str) or not isinstance(source_sentence, str): - return False - - input_sentence = input_sentence.strip() - source_sentence = source_sentence.strip() - - if not input_sentence or not source_sentence: # handle empty string - return False - - input_words = input_sentence.split() # split without arguments - source_words = source_sentence.split() # split without arguments - - if input_sentence == source_sentence and len(input_words) >= min_same_sentence_len: - if verbose: - print("Exact match found.") - return True - - max_overlap_len = longest_common_subsequence(input_words, source_words) - if verbose: - print(f"Max overlap length: {max_overlap_len}") # print overlap length - if max_overlap_len >= min_phrase_sentence_len: - return True - - return False - -def extract_text(url, newspapers = False): - """ - Extracts text from a URL, handling HTML and potential errors. - - Args: - url: The URL of the web page to extract text from. - - Returns: - The extracted text content from the web page, or None if extraction fails. - """ - if newspapers is True: - try: - response = requests.get(url) - response.raise_for_status() # Raise exception for unsuccessful requests - except requests.exceptions.RequestException as e: - print(f"Error fetching URL: {e}") - return None - - try: - news = article(url=url, fetch_images=False) - except: # (ArticleException, ArticleBinaryDataException) as e: - print(f"\t\t↑↑↑ Error downloading article.") - #print(f"\t\t↑↑↑ Error downloading article: {e}") - return None - - return news.text - else: - """ - Extracts text from an HTML page. - """ - response = requests.get(url) - response.raise_for_status() - - response.encoding = response.apparent_encoding - - try: - soup = BeautifulSoup(response.content, "html.parser") - except: - print(f"Error parsing HTML content from {url}") - return None - - # Exclude text within specific elements - for element in soup(["img", "figcaption", "table", "script", "style"]): - element.extract() - #text = soup.get_text(separator="\n") - paragraphs = soup.find_all('p') - text = ' '.join([p.get_text() for p in paragraphs]) - - # remove ", external" which appear after the embedded text - # text = re.sub(r', external', '', text) - - return text - -def check_paraphrase(input_text, page_text, verbose=False): - """ - Checks if the input text is paraphrased in the content at the given URL. - - Args: - input_text: The text to check for paraphrase. - url: The URL of the web page to compare with. - verbose: If True, print debug information. - - Returns: - A tuple containing: - - is_paraphrase: True if the input text is considered a paraphrase, False otherwise. - - paraphrase_results: A list of dictionaries, each containing: - - input_sentence: The sentence from the input text. - - matched_sentence: The corresponding sentence from the web page (if found). - - similarity: The cosine similarity score between the sentences. - - is_paraphrase_sentence: True if the individual sentence pair meets the paraphrase criteria, False otherwise. - """ - is_paraphrase_text = False - - if not isinstance(input_text, str) or not isinstance(page_text, str): - return False, [] - - # Extract sentences from input text and web page - #input_text = remove_punctuation(input_text) - input_sentences = split_into_sentences(input_text) - - - if not page_text: - return is_paraphrase_text, [] - #page_text = remove_punctuation(page_text) - page_sentences = split_into_sentences(page_text) - - if not input_sentences or not page_sentences: - return is_paraphrase_text, [] - - additional_sentences = [] - for sentence in page_sentences: - if ", external" in sentence: - additional_sentences.append(sentence.replace(", external", "")) - page_sentences.extend(additional_sentences) - - min_matching_sentences = math.ceil(len(input_sentences) * MIN_RATIO_PARAPHRASE_NUM) - - # Encode sentences into embeddings - embeddings1 = PARAPHASE_MODEL.encode(input_sentences, convert_to_tensor=True, device=DEVICE) - embeddings2 = PARAPHASE_MODEL.encode(page_sentences, convert_to_tensor=True, device=DEVICE) - - # Compute cosine similarity matrix - similarity_matrix = util.cos_sim(embeddings1, embeddings2).cpu().numpy() - - # Find sentence alignments - alignment = [] - paraphrased_sentence_count = 0 - for i, sentence1 in enumerate(input_sentences): - max_sim_index = np.argmax(similarity_matrix[i]) - max_similarity = similarity_matrix[i][max_sim_index] - - is_paraphrase_sentence = max_similarity > PARAPHRASE_THRESHOLD - - if 0.80 < max_similarity < 0.99: - print(f"\t\tinput_sentence : {sentence1}") - print(f"\t\tmatched_sentence: {page_sentences[max_sim_index]}") - print(f"\t\t--> similarity: {max_similarity}\n") - item = { - "input_sentence": sentence1, - "matched_sentence": page_sentences[max_sim_index], - "similarity": max_similarity, - "is_paraphrase_sentence": is_paraphrase_sentence, - } - - # Check for individual sentence paraphrase if overall paraphrase not yet found - if not is_paraphrase_text and check_sentence( - sentence1, page_sentences[max_sim_index], MIN_SAME_SENTENCE_LEN, MIN_PHRASE_SENTENCE_LEN - ): - is_paraphrase_text = True - if verbose: - print(f"Paraphrase found for individual sentence: {sentence1}") - print(f"Matched sentence: {page_sentences[max_sim_index]}") - - alignment.append(item) - paraphrased_sentence_count += 1 if is_paraphrase_sentence else 0 - - # Check if enough sentences are paraphrases - print (f"\t\tparaphrased_sentence_count: {paraphrased_sentence_count}, min_matching_sentences: {min_matching_sentences}, total_sentence_count: {len(input_sentences)}") - is_paraphrase_text = paraphrased_sentence_count >= min_matching_sentences - - if verbose: - print(f"Minimum matching sentences required: {min_matching_sentences}") - print(f"Total input sentences: {len(input_sentences)}") - print(f"Number of matching sentences: {paraphrased_sentence_count}") - print(f"Is paraphrase: {is_paraphrase_text}") - for item in alignment: - print(item) - - return is_paraphrase_text, alignment - -def similarity_ratio(a, b): - """ - Calculates the similarity ratio between two strings using SequenceMatcher. - - Args: - a: The first string. - b: The second string. - - Returns: - A float representing the similarity ratio between 0.0 and 1.0. - Returns 0.0 if either input is None or not a string. - """ - if not isinstance(a, str) or not isinstance(b, str) or a is None or b is None: - return 0.0 # Handle cases where inputs are not strings or None - return SequenceMatcher(None, a, b).ratio() - - -def is_human_written(sentence): - # 1. Search for exact matches before 2020 - query = f'"{sentence}"' - results = search_by_google(query) - #results = search_bing(sentence) - - # print("\n----------------------\n") - # print(f"# Search results:\n") - # display_Google_results(results) - - if results: - # Exact match found, likely human-written - #return f"human-written\nExact match found: '{sentence}'" - return -1 - - # 2. If no exact match, find similar sentences - query = sentence - results = search_by_google(query) - - if results: - # Check similarity with search results - similarities = [similarity_ratio(sentence, result['snippet']) for result in results] - max_similarity = max(similarities) - - # You can adjust this threshold as needed - if max_similarity > 0.8: - #return f"likely human-written\nFound result that has {max_similarity*100}% of '{sentence}'" - return max_similarity - - # No strong evidence of human authorship - #return f"likely machine-generated\nFound result that has less than 80% similarity of '{sentence}'" - return 1 - -# # Example usage -# sentence = important_sentences[0] -# result = is_human_written(sentence) -# print("\n----------------------\n") -# print(f"# Result:\nThe sentence is {result}") - -def get_text_from_csv(filename): - """ - Reads a CSV file and returns a list of strings, - extracting only the second column (assuming it contains the text). - - Args: - filename: The path to the CSV file. - - Returns: - A list of strings containing the text from the second column. - """ - - text_data = [] - with open(filename, 'r') as file: - reader = csv.reader(file) - next(reader, None) # skip the headers - for row in reader: - if len(row) >= 2: # Check if the row has at least two elements - text_data.append(row[1]) - - return text_data - -if __name__ == '__main__': - # paragraph = """ - # Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to natural intelligence displayed by animals including humans. Leading AI textbooks define the field as the study of "intelligent agents": any system that perceives its environment and takes actions that maximize its chance of achieving its goals. Some popular accounts use the term "artificial intelligence" to describe machines that mimic "cognitive" functions that humans associate with the human mind, such as "learning" and "problem solving", however this definition is rejected by major AI researchers. - # """ - - # keywords = get_keywords(paragraph) - # important_sentences = get_important_sentences(paragraph, keywords) - - # print("# Important sentences:") - # for i, sentence in enumerate(important_sentences, 1): - # print(f"{i}. {sentence}") - # print("\n----------------------\n") - - # sentence = important_sentences[0] - - filename = "data/results/[res]unchanged_words.csv" # Replace with the actual filename - text_list = get_text_from_csv(filename) - - count = 1 - match_count = 0 - unmatch_count = 0 - initial_delay = 1 # second - data = [] - - for text in text_list: - cleaned_text = clean_text(text) - - result = is_human_written(cleaned_text) - match = "match" if result == -1 else "unmatch" - print(f"{count}: [{match}] {text}") - data.append([match, text]) - if result == -1: - match_count += 1 - else: - unmatch_count += 1 - count += 1 - time.sleep(initial_delay) # avoid 100? queries per minute limit - - print(f"Match count: {match_count}") - print(f"Unmatch count: {unmatch_count}") - - df = pd.DataFrame(data, columns=["Text", "Match"]) - output_filename = "data/results/[res]unchanged_words_processed_data.csv" # Specify the output filename - df.to_csv(output_filename, index=False) - - # # Bing search - # subscription_key = "80163c6371fa40e0a50dfaa1dd5b7d84" - # assert subscription_key - # search_url = "https://api.bing.microsoft.com/v7.0/search" - # headers = {"Ocp-Apim-Subscription-Key": subscription_key} - # params = {"q": '"Artificial intelligence (AI) is intelligence demonstrated by machines"', 'freshness': '2000-02-01..2020-02-01', 'answerCount': 2, 'mkt': 'en-US' } - # response = requests.get(search_url, headers=headers, params=params) - # response.raise_for_status() - # search_results = response.json() - # print("\nHeaders:\n") - # print(response.headers) - - # print("\nJSON Response:\n") - # pprint(response.json()) - - - diff --git a/src/texts/Search_Text/test.py b/src/texts/Search_Text/test.py deleted file mode 100644 index 76b3ae9e6e7e8304007be2f2db0e083c2c6563f7..0000000000000000000000000000000000000000 --- a/src/texts/Search_Text/test.py +++ /dev/null @@ -1,38 +0,0 @@ -import re -from bs4 import BeautifulSoup -from newspaper import article, ArticleException -import pandas as pd -import requests -from sentence_transformers import SentenceTransformer, util -from search_text import DEVICE, PARAPHASE_MODEL, extract_text - -#news = article('https://www.bbc.co.uk/news/education-51094279') -#print(news.text) - -def extract_human_data(file_path): - df = pd.read_csv(file_path) - machine_data = df[df["src"] == "xsum_human"] - - # write to file - machine_data.to_csv("data/test_data/MAGE_xsum_human.csv", index=False) - -def connect_lines_without_dot_regex(text): - """Connects lines without dot using regex""" - if not isinstance(text, str): - return text - return re.sub(r'(?