Create app.py

app.py

@@ -0,0 +1,190 @@
+import gradio as gr
+import numpy as np
+import torch
+from PIL import Image, ImageFilter
+import cv2
+from transformers import pipeline
+
+# Set device to GPU if available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"Using device: {device}")
+
+# Load models only once at startup to improve performance
+segmentation_model = "facebook/sam-vit-huge"
+depth_model = "depth-anything/Depth-Anything-V2-Small-hf"
+
+# Initialize pipelines
+segmentation_pipe = pipeline("image-segmentation", model=segmentation_model)
+depth_pipe = pipeline("depth-estimation", model=depth_model)
+
+def get_segmentation_mask(input_image):
+    """Get segmentation mask using the pre-loaded segmentation pipeline"""
+    # Resize image to 512x512 for consistent processing
+    input_image = input_image.resize((512, 512)).convert('RGB')
+    
+    # Get the segmentation result
+    result = segmentation_pipe(input_image)
+    
+    # Extract the first mask (assuming it's the most prominent object)
+    if len(result) > 0:
+        # For SAM-like models that return multiple masks
+        mask = result[0]['mask']
+        mask = np.array(mask) * 255  # Scale to [0, 255]
+    else:
+        # Fallback - create empty mask
+        mask = np.zeros((512, 512), dtype=np.uint8)
+    
+    # Convert to PIL Image
+    mask_img = Image.fromarray(mask.astype(np.uint8))
+    
+    return mask_img, input_image
+
+def apply_background_blur(original_image, mask_image, sigma=15):
+    """Apply Gaussian blur to the background using a segmentation mask"""
+    # Ensure mask is binary (0 for background, 255 for foreground)
+    mask_array = np.array(mask_image)
+    _, binary_mask = cv2.threshold(mask_array, 127, 255, cv2.THRESH_BINARY)
+    mask_img = Image.fromarray(binary_mask)
+    
+    # Create a blurred version of the original image
+    blurred_img = original_image.filter(ImageFilter.GaussianBlur(radius=sigma))
+    
+    # Convert images to numpy arrays for easier manipulation
+    original_array = np.array(original_image)
+    blurred_array = np.array(blurred_img)
+    mask_array = np.array(mask_img)
+    
+    # Create the composite image: foreground from original, background from blurred
+    result_array = np.zeros_like(original_array)
+    
+    # Where mask is white (255), use original image; where mask is black (0), use blurred image
+    for c in range(3):  # For each color channel (RGB)
+        result_array[:, :, c] = np.where(mask_array == 255, 
+                                         original_array[:, :, c], 
+                                         blurred_array[:, :, c])
+    
+    # Convert back to PIL Image
+    result_img = Image.fromarray(result_array)
+    
+    return result_img
+
+def get_depth_map(input_image):
+    """Get depth map using the pre-loaded depth estimation pipeline"""
+    # Ensure image is in RGB format and resized to 512x512
+    input_image = input_image.resize((512, 512)).convert('RGB')
+    
+    # Get the depth map
+    result = depth_pipe(input_image)
+    depth_map = result["depth"]
+    
+    # Convert to numpy array for further processing
+    depth_array = np.array(depth_map)
+    
+    return depth_map, depth_array
+
+def apply_depth_based_blur(original_image, depth_array, max_blur=30):
+    """Apply variable Gaussian blur based on depth"""
+    # Convert depth array to proper format if needed
+    if len(depth_array.shape) == 3 and depth_array.shape[2] > 1:
+        # If depth map has multiple channels, convert to grayscale
+        depth_array = np.mean(depth_array, axis=2)
+    
+    # Normalize depth values to range [0, 1]
+    depth_min = depth_array.min()
+    depth_max = depth_array.max()
+    normalized_depth = (depth_array - depth_min) / (depth_max - depth_min)
+    
+    # Create a series of increasingly blurred versions of the image
+    blurred_images = []
+    for blur_amount in range(max_blur + 1):
+        blurred_images.append(original_image.filter(ImageFilter.GaussianBlur(radius=blur_amount)))
+    
+    # Convert to numpy arrays for easier processing
+    original_array = np.array(original_image)
+    result_array = np.zeros_like(original_array)
+    
+    # For each pixel, determine the blur level based on depth
+    height, width = normalized_depth.shape
+    for y in range(height):
+        for x in range(width):
+            # Calculate blur radius proportional to depth
+            # Higher normalized_depth = farther object = more blur
+            blur_radius = int(normalized_depth[y, x] * max_blur)
+            result_array[y, x] = np.array(blurred_images[blur_radius])[y, x]
+    
+    return Image.fromarray(result_array)
+
+def process_image(input_image, blur_sigma=15, max_depth_blur=30):
+    """Main function to process the image through all effects"""
+    if input_image is None:
+        return None, None, None, None
+    
+    # Resize input image for consistent processing
+    input_image = Image.fromarray(input_image).convert('RGB')
+    input_image = input_image.resize((512, 512))
+    
+    # Step 1: Get segmentation mask
+    mask, _ = get_segmentation_mask(input_image)
+    
+    # Step 2: Apply background blur
+    blurred_background = apply_background_blur(input_image, mask, sigma=blur_sigma)
+    
+    # Step 3: Get depth map
+    depth_map, depth_array = get_depth_map(input_image)
+    
+    # Step 4: Apply depth-based blur
+    depth_blur = apply_depth_based_blur(input_image, depth_array, max_blur=max_depth_blur)
+    
+    # Convert all PIL images to numpy arrays for Gradio
+    input_np = np.array(input_image)
+    mask_np = np.array(mask)
+    blurred_np = np.array(blurred_background)
+    depth_map_np = np.array(depth_map)
+    depth_blur_np = np.array(depth_blur)
+    
+    return input_np, mask_np, blurred_np, depth_map_np, depth_blur_np
+
+# Create Gradio Interface
+with gr.Blocks(title="Image Blur Effects - EEE 515 Assignment 3") as demo:
+    gr.Markdown("# Image Blur Effects App")
+    gr.Markdown("Upload an image to apply segmentation-based blur and depth-based lens blur effects")
+    
+    with gr.Row():
+        input_image = gr.Image(label="Upload Image", type="numpy")
+        
+    with gr.Row():
+        blur_sigma = gr.Slider(minimum=1, maximum=30, value=15, step=1, label="Background Blur Strength (σ)")
+        depth_blur_max = gr.Slider(minimum=1, maximum=50, value=30, step=1, label="Max Depth Blur Strength")
+    
+    with gr.Row():
+        process_btn = gr.Button("Process Image")
+    
+    with gr.Tab("Segmentation Results"):
+        with gr.Row():
+            original_output = gr.Image(label="Original Image", type="numpy")
+            mask_output = gr.Image(label="Segmentation Mask", type="numpy")
+        with gr.Row():
+            blurred_output = gr.Image(label="Background Blur Effect", type="numpy")
+    
+    with gr.Tab("Depth Results"):
+        with gr.Row():
+            depth_map_output = gr.Image(label="Depth Map", type="numpy")
+            depth_blur_output = gr.Image(label="Depth-Based Lens Blur", type="numpy")
+    
+    process_btn.click(
+        fn=process_image,
+        inputs=[input_image, blur_sigma, depth_blur_max],
+        outputs=[original_output, mask_output, blurred_output, depth_map_output, depth_blur_output]
+    )
+    
+    gr.Markdown("## How it works")
+    gr.Markdown("""
+    1. **Segmentation-Based Blur**: Uses a segmentation model to identify the foreground object, 
+       then applies Gaussian blur only to the background.
+       
+    2. **Depth-Based Lens Blur**: Uses a monocular depth estimation model to create a depth map,
+       then applies varying levels of blur based on the estimated depth.
+    """)
+
+# Launch the app
+demo.launch()