Upload 9 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+capsule_crack.png filter=lfs diff=lfs merge=lfs -text
+hazelnut_cut.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,5 @@
 ---
-title: AnomalyGPT1
-emoji: 😻
-colorFrom: yellow
-colorTo: purple
+license: cc-by-sa-4.0
+title: AnomalyGPT
 sdk: gradio
-sdk_version: 5.25.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---

app.py

@@ -0,0 +1,245 @@
+import os
+
+os.system("cp /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda118.so /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cpu.so")
+
+
+import gradio as gr
+import mdtex2html
+from model.openllama import OpenLLAMAPEFTModel
+import torch
+from io import BytesIO
+from PIL import Image as PILImage
+import cv2
+import numpy as np
+from matplotlib import pyplot as plt
+from torchvision import transforms
+
+# init the model
+args = {
+    'model': 'openllama_peft',
+    'imagebind_ckpt_path': './pretrained_ckpt/imagebind_ckpt/imagebind_huge.pth',
+    'vicuna_ckpt_path': './pretrained_ckpt/vicuna_ckpt/7b_v0',
+    'anomalygpt_ckpt_path': './ckpt/train_supervised/pytorch_model.pt',
+    'delta_ckpt_path': './pretrained_ckpt/pandagpt_ckpt/7b/pytorch_model.pt',
+    'stage': 2,
+    'max_tgt_len': 128,
+    'lora_r': 32,
+    'lora_alpha': 32,
+    'lora_dropout': 0.1
+}
+
+model = OpenLLAMAPEFTModel(**args)
+delta_ckpt = torch.load(args['delta_ckpt_path'], map_location=torch.device('cpu'))
+model.load_state_dict(delta_ckpt, strict=False)
+delta_ckpt = torch.load(args['anomalygpt_ckpt_path'], map_location=torch.device('cpu'))
+model.load_state_dict(delta_ckpt, strict=False)
+model = model.eval()#.half()#.cuda()
+# model.image_decoder = model.image_decoder.cuda()
+# model.prompt_learner = model.prompt_learner.cuda()
+
+"""Override Chatbot.postprocess"""
+def postprocess(self, y):
+    if y is None:
+        return []
+    for i, (message, response) in enumerate(y):
+        y[i] = (
+            None if message is None else mdtex2html.convert((message)),
+            None if response is None else mdtex2html.convert(response),
+        )
+    return y
+
+
+gr.Chatbot.postprocess = postprocess
+
+
+def parse_text(text):
+    """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/"""
+    lines = text.split("\n")
+    lines = [line for line in lines if line != ""]
+    count = 0
+    for i, line in enumerate(lines):
+        if "```" in line:
+            count += 1
+            items = line.split('`')
+            if count % 2 == 1:
+                lines[i] = f'<pre><code class="language-{items[-1]}">'
+            else:
+                lines[i] = f'<br></code></pre>'
+        else:
+            if i > 0:
+                if count % 2 == 1:
+                    line = line.replace("`", "\`")
+                    line = line.replace("<", "&lt;")
+                    line = line.replace(">", "&gt;")
+                    line = line.replace(" ", "&nbsp;")
+                    line = line.replace("*", "&ast;")
+                    line = line.replace("_", "&lowbar;")
+                    line = line.replace("-", "&#45;")
+                    line = line.replace(".", "&#46;")
+                    line = line.replace("!", "&#33;")
+                    line = line.replace("(", "&#40;")
+                    line = line.replace(")", "&#41;")
+                    line = line.replace("$", "&#36;")
+                lines[i] = "<br>"+line
+    text = "".join(lines)
+    return text
+
+
+def predict(
+    input, 
+    image_path, 
+    normal_img_path,  
+    chatbot, 
+    max_length, 
+    top_p, 
+    temperature, 
+    history, 
+    modality_cache, 
+):
+    
+    if image_path is None and normal_img_path is None:
+        return [(input, "There is no input data provided! Please upload your data and start the conversation.")]
+    else:
+        print(f'[!] image path: {image_path}\n[!] normal image path: {normal_img_path}\n')
+
+    # prepare the prompt
+    prompt_text = ''
+    for idx, (q, a) in enumerate(history):
+        if idx == 0:
+            prompt_text += f'{q}\n### Assistant: {a}\n###'
+        else:
+            prompt_text += f' Human: {q}\n### Assistant: {a}\n###'
+    if len(history) == 0:
+        prompt_text += f'{input}'
+    else:
+        prompt_text += f' Human: {input}'
+
+    response, pixel_output = model.generate({
+        'prompt': prompt_text,
+        'image_paths': [image_path] if image_path else [],
+        'normal_img_paths': [normal_img_path] if normal_img_path else [],
+        'audio_paths': [],
+        'video_paths': [],
+        'thermal_paths': [],
+        'top_p': top_p,
+        'temperature': temperature,
+        'max_tgt_len': max_length,
+        'modality_embeds': modality_cache
+    },web_demo=True)
+    chatbot.append((parse_text(input), parse_text(response)))
+    history.append((input, response))
+
+
+    plt.imshow(pixel_output.to(torch.float16).reshape(224,224).detach().cpu(), cmap='binary_r')
+    plt.axis('off')
+    plt.savefig('output.png',bbox_inches='tight',pad_inches = 0)
+
+    target_size = 435
+    original_width, original_height = PILImage.open(image_path).size
+    if original_width > original_height:
+        new_width = target_size
+        new_height = int(target_size * (original_height / original_width))
+    else:
+        new_height = target_size
+        new_width = int(target_size * (original_width / original_height))
+
+    new_image = PILImage.new('L', (target_size, target_size), 255)  # 'L' mode for grayscale
+
+    paste_x = (target_size - new_width) // 2
+    paste_y = (target_size - new_height) // 2
+
+    pixel_output = PILImage.open('output.png').resize((new_width, new_height), PILImage.LANCZOS)
+
+    new_image.paste(pixel_output, (paste_x, paste_y))
+
+    new_image.save('output.png')
+
+    image = cv2.imread('output.png', cv2.IMREAD_GRAYSCALE)
+    kernel = np.ones((3, 3), np.uint8)
+    eroded_image = cv2.erode(image, kernel, iterations=1)
+    cv2.imwrite('output.png', eroded_image)
+
+    output =  PILImage.open('output.png').convert('L')
+
+
+    return chatbot, history, modality_cache, output
+
+
+
+def reset_user_input():
+    return gr.update(value='')
+
+
+def reset_state():
+    return gr.update(value=''), None, None, [], [], [], PILImage.open('ffffff.png')
+
+examples = ['hazelnut_cut.png','capsule_crack.png','carpet_normal.jpg']
+
+with gr.Blocks() as demo:
+    gr.HTML("""<h1 align="center">Demo of AnomalyGPT</h1>""")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            with gr.Row():
+                image_path = gr.Image(type="filepath", label="Query Image", value=examples[0])
+            with gr.Row():
+                normal_img_path = gr.Image(type="filepath", label="Normal Image (optional)", value=None)
+            with gr.Row():
+                gr.Examples(examples=examples, inputs=[image_path])
+            with gr.Row():
+                max_length = gr.Slider(0, 512, value=512, step=1.0, label="Max length", interactive=True)
+                top_p = gr.Slider(0, 1, value=0.01, step=0.01, label="Top P", interactive=True)
+                temperature = gr.Slider(0, 1, value=1.0, step=0.01, label="Temperature", interactive=True)
+
+
+        with gr.Column(scale=3):
+            with gr.Row():
+                with gr.Column(scale=6):
+                    chatbot = gr.Chatbot().style(height=440)
+                with gr.Column(scale=4):
+                    # gr.Image(output)
+                    image_output = gr.Image(interactive=False, label="Localization Output", type='pil',value=PILImage.open('ffffff.png'))
+            with gr.Row():
+                user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=12).style(container=False)
+            with gr.Row():
+                with gr.Column(scale=2):
+                    submitBtn = gr.Button("Submit", variant="primary")
+                with gr.Column(scale=1):
+                    emptyBtn = gr.Button("Clear History")
+                    
+    history = gr.State([])
+    modality_cache = gr.State([])
+
+    submitBtn.click(
+        predict, [
+            user_input, 
+            image_path, 
+            normal_img_path, 
+            chatbot, 
+            max_length, 
+            top_p, 
+            temperature, 
+            history, 
+            modality_cache,
+        ], [
+            chatbot, 
+            history,
+            modality_cache,
+            image_output
+        ],
+        show_progress=True
+    )
+
+    submitBtn.click(reset_user_input, [], [user_input])
+    emptyBtn.click(reset_state, outputs=[
+        user_input,
+        image_path,
+        normal_img_path,
+        chatbot, 
+        history, 
+        modality_cache,
+        image_output
+    ], show_progress=True)
+
+
+demo.queue().launch()

gitattributes

@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+.bin filter=lfs diff=lfs merge=lfs -text
+.pt filter=lfs diff=lfs merge=lfs -text
+.pth filter=lfs diff=lfs merge=lfs -text
+hazelnut_cut.png filter=lfs diff=lfs merge=lfs -text
+capsule_crack.png filter=lfs diff=lfs merge=lfs -text

header.py

@@ -0,0 +1,35 @@
+import torch
+import datetime
+import types
+import deepspeed
+from transformers.deepspeed import HfDeepSpeedConfig
+import transformers
+import numpy as np
+from collections import OrderedDict
+from torch.utils.data import Dataset, DataLoader
+from torch.nn.utils import clip_grad_norm_
+from torch.cuda.amp import autocast, GradScaler
+from torch.nn import DataParallel
+from torch.optim import lr_scheduler
+import torch.optim as optim
+import torch.nn as nn
+import torch.nn.functional as F
+from tqdm import tqdm
+import os
+import re
+import math
+import random
+import json
+import time
+import logging
+from copy import deepcopy
+import ipdb
+import argparse
+from model.ImageBind import data
+from transformers import LlamaTokenizer, LlamaForCausalLM, LlamaConfig
+from torch.nn.utils.rnn import pad_sequence
+from peft import LoraConfig, TaskType, get_peft_model
+
+logging.getLogger("transformers").setLevel(logging.WARNING)
+logging.getLogger("transformers.tokenization_utils").setLevel(logging.ERROR)
+os.environ['TOKENIZERS_PARALLELISM'] = 'false'

requirements.txt

@@ -0,0 +1,29 @@
+deepspeed==0.9.2
+easydict==1.10
+einops==0.6.1
+ftfy==6.1.1
+gradio==3.41.2
+h5py==3.9.0
+iopath==0.1.10
+ipdb==0.13.13
+kornia==0.7.0
+matplotlib==3.7.2
+mdtex2html==1.2.0
+numpy==1.24.3
+open3d_python==0.3.0.0
+opencv_python==4.8.0.74
+peft==0.3.0
+Pillow==10.0.0
+pytorchvideo==0.1.5
+PyYAML==6.0.1
+regex==2022.10.31
+timm==0.6.7
+torch==1.13.1
+torchaudio==0.13.1
+torchvision==0.14.1
+tqdm==4.64.1
+transformers==4.30.2
+sentencepiece
+accelerate==0.21.0
+bitsandbytes==0.41.1
+scipy