v1

.gitattributes

@@ -2,13 +2,11 @@
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
 *.npy filter=lfs diff=lfs merge=lfs -text
@@ -16,20 +14,21 @@
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
 *.pickle filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

SDXL_inpaint.ipynb

@@ -0,0 +1,475 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU",
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "d73ad5cc5b1c45e5bac35da4b105d7d4": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_c4ce5cafe6e84a6cb8a3e9a043c67913",
+              "IPY_MODEL_d0e4bf336ad241958372f92c3fc6cd48",
+              "IPY_MODEL_4f101488783446bfa337e6d4809f9ca5"
+            ],
+            "layout": "IPY_MODEL_a664851d48084d5a8bed4efb15e3eecf"
+          }
+        },
+        "c4ce5cafe6e84a6cb8a3e9a043c67913": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_0c8774c074194255961305239d428887",
+            "placeholder": "​",
+            "style": "IPY_MODEL_2d71384e54a94332ae4c21f48fe404d1",
+            "value": "100%"
+          }
+        },
+        "d0e4bf336ad241958372f92c3fc6cd48": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_a30a7bd69266403aa5774d3045087447",
+            "max": 49,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_7edfc44057194cdb8880a1544cf13d3e",
+            "value": 49
+          }
+        },
+        "4f101488783446bfa337e6d4809f9ca5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_d332c661fc54456c9e007a6257c08427",
+            "placeholder": "​",
+            "style": "IPY_MODEL_9302974855bc4f25bc4e5239e87fbfc3",
+            "value": " 49/49 [00:37<00:00,  1.23it/s]"
+          }
+        },
+        "a664851d48084d5a8bed4efb15e3eecf": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "0c8774c074194255961305239d428887": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "2d71384e54a94332ae4c21f48fe404d1": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "a30a7bd69266403aa5774d3045087447": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "7edfc44057194cdb8880a1544cf13d3e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "d332c661fc54456c9e007a6257c08427": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "9302974855bc4f25bc4e5239e87fbfc3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        }
+      }
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install gradio"
+      ],
+      "metadata": {
+        "id": "sFIrvgM6Zz9i"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "k5XlAek1Zwpz"
+      },
+      "outputs": [],
+      "source": [
+        "import torch\n",
+        "from diffusers import AutoPipelineForInpainting\n",
+        "from PIL import Image\n",
+        "\n",
+        "# Check for GPU availability\n",
+        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+        "\n",
+        "# Load the inpainting pipeline\n",
+        "pipe = AutoPipelineForInpainting.from_pretrained(\n",
+        "    \"diffusers/stable-diffusion-xl-1.0-inpainting-0.1\",\n",
+        "    torch_dtype=torch.float16,  # Use float16 for efficiency\n",
+        "    variant=\"fp16\"\n",
+        ").to(device)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load input images\n",
+        "background_image = Image.open(\"woman.png\").convert(\"RGB\")  # Ensure RGB format\n",
+        "mask_image = Image.open(\"mask.png\").convert(\"L\")  # Ensure grayscale format\n",
+        "# Resize the mask to match the background image size\n",
+        "mask_image = mask_image.resize(background_image.size, Image.LANCZOS)"
+      ],
+      "metadata": {
+        "id": "JDCRPTsFM3B7"
+      },
+      "execution_count": 6,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "output = pipe(\n",
+        "    prompt=\"A Woman wearing sunglasses, smiling\",\n",
+        "    negative_prompt=\"ugly, low quality, blur, distorted, unreal\",\n",
+        "    image=background_image,\n",
+        "    mask_image=mask_image,\n",
+        "    guidance_scale=7.5,\n",
+        "    num_inference_steps=50,\n",
+        "    strength=0.99,\n",
+        "    height=background_image.size[1],\n",
+        "    width=background_image.size[0]\n",
+        ")\n",
+        "\n",
+        "\n",
+        "# Save and display output\n",
+        "inpainted_image = output.images[0]\n",
+        "inpainted_image.save(\"output.png\")\n",
+        "inpainted_image.show()\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 49,
+          "referenced_widgets": [
+            "d73ad5cc5b1c45e5bac35da4b105d7d4",
+            "c4ce5cafe6e84a6cb8a3e9a043c67913",
+            "d0e4bf336ad241958372f92c3fc6cd48",
+            "4f101488783446bfa337e6d4809f9ca5",
+            "a664851d48084d5a8bed4efb15e3eecf",
+            "0c8774c074194255961305239d428887",
+            "2d71384e54a94332ae4c21f48fe404d1",
+            "a30a7bd69266403aa5774d3045087447",
+            "7edfc44057194cdb8880a1544cf13d3e",
+            "d332c661fc54456c9e007a6257c08427",
+            "9302974855bc4f25bc4e5239e87fbfc3"
+          ]
+        },
+        "id": "u4KLSfGPNAZK",
+        "outputId": "f63805bc-5315-4541-8633-e3ddcc9bc4fd"
+      },
+      "execution_count": 10,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "  0%|          | 0/49 [00:00<?, ?it/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "d73ad5cc5b1c45e5bac35da4b105d7d4"
+            }
+          },
+          "metadata": {}
+        }
+      ]
+    }
+  ]
+}

app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+import numpy as np
+import torch
+import logging
+import os
+import cv2 
+from huggingface_hub import login
+from diffusers import AutoPipelineForInpainting, UNet2DConditionModel
+import diffusers
+import spaces # type: ignore
+from PIL import Image
+
+login(os.getenv("HF_TOKEN"))
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+pipe = AutoPipelineForInpainting.from_pretrained("diffusers/stable-diffusion-xl-1.0-inpainting-0.1", torch_dtype=torch.float16, variant="fp16").to(device, dtype=torch.float16)
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+@spaces.GPU()
+def predict(image, mask=None, positive_prompt="", negative_prompt=None, guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler"):
+    try:
+        logging.info("Starting inpainting process")
+        
+        scheduler_parts = scheduler.split("-")
+        scheduler_class_name = scheduler_parts[0]
+        add_kwargs = {}
+
+        if len(scheduler_parts) > 1:
+            add_kwargs["use_karras"] = True
+        if len(scheduler_parts) > 2:
+            add_kwargs["algorithm_type"] = "sde-dpmsolver++"
+        
+        logging.info(f"Using scheduler: {scheduler_class_name} with args {add_kwargs}")
+        
+        scheduler_cls = getattr(diffusers, scheduler_class_name)
+
+        source_image = Image.fromarray(image["background"]).convert("RGB")
+        
+        if np.unique(mask["background"]).size == 1:
+            alpha_channel = image["layers"][0][:, :, 3]
+            mask = np.where(alpha_channel == 255,255,0).astype(np.uint8)
+            mask_4ch = np.stack([mask] * 4, axis=2)
+            mask_image = Image.fromarray(mask_4ch)
+            logging.info(f"Mask was not uploaded")
+        else :
+            mask_image = Image.fromarray(mask["background"]).convert("RGBA")
+            print(type(mask["background"]))
+            print("Value of Mask : ", mask)
+            print("Value of Mask : ", mask["background"])
+            logging.info(f"Mask was uploaded")
+        
+        
+
+        logging.info("Converted images to required formats")
+
+        # Set the scheduler
+        pipe.scheduler = scheduler_cls.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler", **add_kwargs)
+        logging.info("Scheduler set successfully")
+
+        h = (source_image.height//8) * 8
+        w = (source_image.width//8) * 8
+        # Run the inpainting model
+        output = pipe(
+            prompt=positive_prompt, 
+            negative_prompt=negative_prompt, 
+            image=source_image, 
+            height=h,
+            width=w,
+            mask_image=mask_image,
+            guidance_scale=guidance_scale, 
+            num_inference_steps=int(steps), 
+            strength=strength
+        )
+        logging.info("Model inference completed successfully")
+
+            
+        return output.images[0], mask_image
+    
+    except Exception as e:
+        logging.error(f"Error occurred: {str(e)}", exc_info=True)
+        return None
+
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            image = gr.ImageMask(
+                type="numpy", label="Input Image", layers=False
+            )
+            with gr.Row():
+                positive_prompt = gr.Textbox(placeholder="Your prompt", label="Positive Prompt")
+                negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="What you don't want to see in the image")
+            with gr.Row():    
+                btn = gr.Button("Inpaint")
+            with gr.Accordion(label="Advanced Settings", open=False):
+                with gr.Row():
+                    guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="Guidance Scale")
+                    steps = gr.Number(value=50, minimum=10, maximum=100, step=1, label="Steps")
+                    strength = gr.Number(value=0.80, minimum=0.01, maximum=1.0, step=0.01, label="Strength")
+                    
+                with gr.Row():
+                    schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"]
+                    scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="DPMSolverMultistepScheduler-Karras-SDE")
+                with gr.Row():
+                    mask = gr.ImageMask(label="Mask", format="png", value=None, sources=["upload"])
+                    mask_image = gr.Image(label="Mask Image", format="png")
+        with gr.Column():
+            image_out = gr.Image(label="Output", format="png")
+    
+    btn.click(fn=predict, inputs=[image, mask, positive_prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, mask_image])
+    
+demo.launch(debug=True,show_error=True)

requirements.txt

@@ -0,0 +1,10 @@
+--extra-index-url https://download.pytorch.org/whl/cu118
+torch
+git+https://github.com/huggingface/diffusers.git
+transformers
+accelerate
+ftfy
+numpy
+matplotlib
+uuid
+opencv-python