import gradio as gr from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL, EulerAncestralDiscreteScheduler import torch import numpy as np import cv2 from PIL import Image import spaces # 🌟 Auto-detect device (CPU/GPU) device = "cuda" precision = torch.float16 # 🏗️ Load ControlNet model for Canny edge detection # xinsir/controlnet-canny-sdxl-1.0 # diffusers/controlnet-canny-sdxl-1.0 controlnet = ControlNetModel.from_pretrained( "xinsir/controlnet-canny-sdxl-1.0", torch_dtype=precision ) # when test with other base model, you need to change the vae also. vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=precision) # Scheduler eulera_scheduler = EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler") # Stable Diffusion Model pipe = StableDiffusionXLControlNetPipeline.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, vae=vae, torch_dtype=precision, scheduler=eulera_scheduler, ) # Load lora (giving it a name makes it active when using the name in the prompt) pipe.load_lora_weights("ostris/ikea-instructions-lora-sdxl", weight_name="ikea_instructions_xl_v1_5.safetensors", adapter_name="ikea") pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") pipe.load_lora_weights('e-n-v-y/envy-junkworld-xl-01', weight_name='EnvyJunkworldXL01.safetensors', adapter_name="junkworld") pipe.disable_lora() def activate_ikea_lora(): print("Activating IKEA LoRa") pipe.disable_lora() while pipe.get_active_adapters()[0] != "ikea": pipe.set_adapters("ikea") pipe.enable_lora() print("IKEA LoRa active!") def activate_pixel_lora(): print("Activating PixelArt LoRa") pipe.disable_lora() while pipe.get_active_adapters()[0] != "pixel": pipe.set_adapters("pixel") pipe.enable_lora() print("PixelArt LoRa active!") def activate_junkworld_lora(): print("Activating JunkWorld LoRa") pipe.disable_lora() while pipe.get_active_adapters()[0] != "junkworld": pipe.set_adapters("junkworld") pipe.enable_lora() print("JunkWorld LoRa active!") def disable_loras(): print("Deactivating LoRas") pipe.disable_lora() print("All LoRas deactivated!") pipe.to(device) # 📸 Edge detection function using OpenCV (Canny) @spaces.GPU def apply_canny(image, low_threshold, high_threshold): image = np.array(image) image = cv2.Canny(image, low_threshold, high_threshold) image = image[:, :, None] image = np.concatenate([image, image, image], axis=2) return Image.fromarray(image) # 🎨 Image generation function @spaces.GPU def generate_image(prompt, input_image, low_threshold, high_threshold, strength, guidance, controlnet_conditioning_scale): print(pipe.get_active_adapters()) # Apply edge detection edge_detected = apply_canny(input_image, low_threshold, high_threshold) # Generate styled image using ControlNet result = pipe( prompt=prompt, image=edge_detected, num_inference_steps=30, guidance_scale=guidance, controlnet_conditioning_scale=float(controlnet_conditioning_scale), strength=strength ).images[0] return edge_detected, result # 🖥️ Gradio UI with gr.Blocks() as demo: gr.Markdown("# 🏗️ 3D Screenshot to Styled Render with ControlNet") with gr.Row(): with gr.Column(): input_image = gr.Image(label="Upload 3D Screenshot", type="pil") prompt = gr.Textbox(label="Style Prompt", placeholder="e.g., Futuristic building in sunset") low_threshold = gr.Slider(50, 150, value=100, label="Canny Edge Low Threshold") high_threshold = gr.Slider(100, 200, value=150, label="Canny Edge High Threshold") strength = gr.Slider(0.1, 1.0, value=0.7, label="Denoising Strength") guidance = gr.Slider(1, 20, value=7.5, label="Guidance Scale (Creativity)") controlnet_conditioning_scale = gr.Slider(0, 1, value=0.5, step=0.01, label="ControlNet Conditioning Scale") with gr.Row(): ikea_lora_button = gr.Button("IKEA Instructions") pixel_lora_button = gr.Button("Pixel Art") junkworld_lora_button = gr.Button("Junk World") disable_lora_button = gr.Button("Disable LoRas") generate_button = gr.Button("Generate Styled Image") with gr.Column(): edge_output = gr.Image(label="Edge Detected Image") result_output = gr.Image(label="Generated Styled Image") # 🔗 Generate Button Action generate_button.click( fn=generate_image, inputs=[prompt, input_image, low_threshold, high_threshold, strength, guidance, controlnet_conditioning_scale], outputs=[edge_output, result_output] ) ikea_lora_button.click( fn = activate_ikea_lora, ) pixel_lora_button.click( fn = activate_pixel_lora, ) junkworld_lora_button.click( fn = activate_junkworld_lora, ) disable_lora_button.click( fn = disable_loras, ) # 🚀 Launch the app demo.launch()