import gradio as gr import PIL import numpy as np from models.maskclip import MaskClip from models.dino import DINO import torchvision.transforms as T import torch.nn.functional as F from lposs import lposs, lposs_plus import torch import spaces device = "cpu" if torch.cuda.is_available(): print("Using GPU") device = "cuda" # elif torch.backends.mps.is_available(): # device = "mps" print(f"Using device: {device}") maskclip = MaskClip().to(device) dino = DINO().to(device) to_torch_tensor = T.Compose([T.Resize(size=448, max_size=2048), T.ToTensor()]) # Default hyperparameter values DEFAULT_SIGMA = 100 DEFAULT_ALPHA = 0.95 DEFAULT_K = 400 DEFAULT_WSIZE = 224 DEFAULT_GAMMA = 3.0 DEFAULT_TAU = 0.01 DEFAULT_R = 13 # Function to reset hyperparameters to default values def reset_hyperparams(): return DEFAULT_WSIZE, DEFAULT_K, DEFAULT_GAMMA, DEFAULT_ALPHA, DEFAULT_SIGMA, DEFAULT_TAU, DEFAULT_R @spaces.GPU def segment_image(img: PIL.Image.Image, classnames: str, use_lposs_plus: bool | None, winodw_size:int, k:int, gamma:float, alpha:float, sigma: float, tau:float, r:int) -> tuple[np.ndarray | PIL.Image.Image | str, list[tuple[np.ndarray | tuple[int, int, int, int], str]]]: img_tensor = to_torch_tensor(PIL.Image.fromarray(img)).unsqueeze(0).to(device) classnames = [c.strip() for c in classnames.split(",")] num_classes = len(classnames) winodw_size = (winodw_size, winodw_size) stride = (winodw_size[0] // 2, winodw_size[1] // 2) preds = lposs(maskclip, dino, img_tensor, classnames, window_size=winodw_size, window_stride=stride, sigma=1/sigma, lp_k_image=k, lp_gamma=gamma, lp_alpha=alpha) if use_lposs_plus: preds = lposs_plus(img_tensor, preds, tau=tau, alpha=alpha, r=r) preds = F.interpolate(preds, size=img.shape[:-1], mode="bilinear", align_corners=False) preds = F.softmax(preds * 100, dim=1).cpu().numpy() return (img, [(preds[0, i, :, :], classnames[i]) for i in range(num_classes)]) with gr.Blocks() as demo: gr.Markdown("# LPOSS: Label Propagation Over Patches and Pixels for Open-vocabulary Semantic Segmentation") gr.Markdown("""
📄 arXiv 💻 GitHub
""") gr.Markdown("Upload an image and specify the objects you want to segment by listing their names separated by commas.") with gr.Accordion("Hyper-parameters", open=False): with gr.Column(scale=1): # with gr.Row(): # gr.Markdown("Hyper-parameters") with gr.Row(): window_size = gr.Slider(minimum=112, maximum=448, value=DEFAULT_WSIZE, step=16, label="Window Size") k = gr.Slider(minimum=50, maximum=800, value=DEFAULT_K, step=50, label="k (LPOSS number of graph neighbors)") gamma = gr.Slider(minimum=0.0, maximum=10.0, value=DEFAULT_GAMMA, step=0.5, label="γ (LPOSS graph edge tuning)") sigma = gr.Slider(minimum=50, maximum=400, value=DEFAULT_SIGMA, step=10, label="σ (LPOSS spatial affinity tuning)") tau = gr.Slider(minimum=0.0, maximum=1.0, value=DEFAULT_TAU, step=0.01, label="τ (LPOSS+ appearance affinity tuning)") r = gr.Slider(minimum=3, maximum=15, value=DEFAULT_R, step=2, label="r (LPOSS+ kernel size)") alpha = gr.Slider(minimum=0.0, maximum=1.0, value=DEFAULT_ALPHA, step=0.05, label="α (amount of propagation)") with gr.Row(): reset_btn = gr.Button("Reset to Default Values") with gr.Row(): class_names = gr.Textbox(label="Class Names", info="Separate class names with commas") use_lposs_plus = gr.Checkbox(label="Use LPOSS+", info="Enable pixel-level refinement using LPOSS+") with gr.Row(): segment_btn = gr.Button("Segment Image") with gr.Row(): with gr.Column(scale=2): input_image = gr.Image(label="Input Image") # class_names = gr.Textbox(label="Class Names", info="Separate class names with commas") # use_lposs_plus = gr.Checkbox(label="Use LPOSS+", info="Enable pixel-level refinement using LPOSS+") with gr.Column(scale=3): output_image = gr.AnnotatedImage(label="Segmentation Results") reset_btn.click(fn=reset_hyperparams, outputs=[window_size, k, gamma, alpha, sigma, tau, r]) segment_btn.click( fn=segment_image, inputs=[input_image, class_names, use_lposs_plus, window_size, k, gamma, alpha, sigma, tau, r], outputs=[output_image] ) demo.launch()