keypoint examples

app.py

@@ -546,6 +546,12 @@ def reset_kps(img, keypoints, side: Literal["right", "left"]):
         keypoints[1] = []
     return img, keypoints
 
+def read_kpts(kpts_path):
+    if kpts_path is None or len(kpts_path)==0:
+        return None
+    kpts = np.load(kpts_path)
+    return kpts
+
 def stay_crop(img, crop_coord):
     if img is not None:
         if crop_coord is None:
@@ -651,36 +657,35 @@ def sample_diff(ref_cond, target_cond, target_keypts, num_gen, seed, cfg):
     return results, results_pose
 
 @spaces_120_fn
-def ready_sample(img_cropped, inpaint_mask, keypts):
-    # img = cv2.resize(img_ori[..., :3], opts.image_size, interpolation=cv2.INTER_AREA)
+def ready_sample(img_cropped, inpaint_mask, keypts, keypts_np):
     img = cv2.resize(img_cropped["background"][..., :3], opts.image_size, interpolation=cv2.INTER_AREA)
     sam_predictor.set_image(img)
-    if len(keypts[0]) == 0:
-        keypts[0] = np.zeros((21, 2))
-    elif len(keypts[0]) == 21:
-        keypts[0] = np.array(keypts[0], dtype=np.float32)
-        # keypts[0][:, 0] = keypts[0][:, 0] + crop_coord[0][0]
-        # keypts[0][:, 1] = keypts[0][:, 1] + crop_coord[0][1]
-    else:
-        gr.Info("Number of right hand keypoints should be either 0 or 21.")
-        return None, None
-
-    if len(keypts[1]) == 0:
-        keypts[1] = np.zeros((21, 2))
-    elif len(keypts[1]) == 21:
-        keypts[1] = np.array(keypts[1], dtype=np.float32)
-        # keypts[1][:, 0] = keypts[1][:, 0] + crop_coord[0][0]
-        # keypts[1][:, 1] = keypts[1][:, 1] + crop_coord[0][1]
+    if keypts is None and keypts_np is not None:
+        keypts = keypts_np
     else:
-        gr.Info("Number of left hand keypoints should be either 0 or 21.")
-        return None, None
-
-    keypts = np.concatenate(keypts, axis=0)
-    keypts = scale_keypoint(keypts, (img_cropped["background"].shape[1], img_cropped["background"].shape[0]), opts.image_size)
+        if len(keypts[0]) == 0:
+            keypts[0] = np.zeros((21, 2))
+        elif len(keypts[0]) == 21:
+            keypts[0] = np.array(keypts[0], dtype=np.float32)
+            # keypts[0][:, 0] = keypts[0][:, 0] + crop_coord[0][0]
+            # keypts[0][:, 1] = keypts[0][:, 1] + crop_coord[0][1]
+        else:
+            gr.Info("Number of right hand keypoints should be either 0 or 21.")
+            return None, None
+        if len(keypts[1]) == 0:
+            keypts[1] = np.zeros((21, 2))
+        elif len(keypts[1]) == 21:
+            keypts[1] = np.array(keypts[1], dtype=np.float32)
+            # keypts[1][:, 0] = keypts[1][:, 0] + crop_coord[0][0]
+            # keypts[1][:, 1] = keypts[1][:, 1] + crop_coord[0][1]
+        else:
+            gr.Info("Number of left hand keypoints should be either 0 or 21.")
+            return None, None
+        keypts = np.concatenate(keypts, axis=0)
+        keypts = scale_keypoint(keypts, (img_cropped["background"].shape[1], img_cropped["background"].shape[0]), opts.image_size)
 
     box_shift_ratio = 0.5
     box_size_factor = 1.2
-
     if keypts[0].sum() != 0 and keypts[21].sum() != 0:
         input_point = np.array(keypts)
         input_box = np.stack([keypts.min(axis=0), keypts.max(axis=0)])
@@ -808,7 +813,7 @@ def sample_inpaint(
     cfg,
     quality,
 ):
-    if keypts is None:
+    if inpaint_latent_mask is None:
         return None, None, None
     set_seed(seed)
     N = num_gen
@@ -830,6 +835,7 @@ def sample_inpaint(
         cfg_scale=cfg_scale,
     )
 
+    gr.Info("The process successfully started to run. Please wait for around 3.5 minutes.", duration=20)
     samples, _ = diffusion.inpaint_p_sample_loop(
         model.forward_with_cfg,
         z.shape,
@@ -977,8 +983,9 @@ def fix_clear_all():
         None,
         None,
         None,
+        None,
+        None,
         1,
-        # (0,0),
         42,
         3.0,
         10,
@@ -1216,6 +1223,20 @@ fix_example_brush = [
     # ["bad_hands/14_mask.jpg"],
     # ["bad_hands/15_mask.jpg"],
 ]
+fix_example_kpts = [
+    ["bad_hands/1_kpts.png"],
+    ["bad_hands/3_kpts.png"],
+    ["bad_hands/4_kpts.png"],
+    ["bad_hands/5_kpts.png"],
+    ["bad_hands/6_kpts.png"],
+    ["bad_hands/7_kpts.png"],
+]
+for i in range(len(fix_example_kpts)):
+    npy_path = fix_example_kpts[i][0].replace("_kpts.png", ".npy")
+    # kpts = np.load(npy_path)
+    # kpts = gr.State(kpts)
+    fix_example_kpts[i].append(npy_path)
+
 custom_css = """
 .gradio-container .examples img {
     width: 240px !important;
@@ -1231,6 +1252,10 @@ custom_css = """
     font-weight: bold !important;
     background-color: #90EE90 !important;
 }
+#kpts_examples table tr th:nth-child(2),
+#kpts_examples table tr td:nth-child(2) {
+    display: none !important;
+}
 """
 #    color: black !important;
 
@@ -1290,6 +1315,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         fix_crop_coord = gr.State(value=None)
         fix_img = gr.State(value=None)
         fix_kpts = gr.State(value=None)
+        fix_kpts_path = gr.Textbox(visible=False)
         fix_kpts_np = gr.State(value=None)
         fix_ref_cond = gr.State(value=None)
         fix_target_cond = gr.State(value=None)
@@ -1390,73 +1416,93 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
             # keypoint selection
             with gr.Column():
                 gr.Markdown(
-                    """<p style="text-align: center; font-size: 18px; font-weight: bold;">3. Click on hand to get target hand pose</p>"""
-                )
-                gr.Markdown(
-                    """<p style="text-align: center;">&#9312; Tell us if this is right, left, or both hands</p>"""
+                    """<p style="text-align: center; font-size: 18px; font-weight: bold;">3. Target hand pose</p>"""
                 )
-                fix_checkbox = gr.CheckboxGroup(
-                    ["Right hand", "Left hand"],
-                    show_label=False,
-                    interactive=False,
-                )
-                fix_kp_r_info = gr.Markdown(
-                    """<p style="text-align: center;">&#9313; Click 21 keypoints on the image to provide the target hand pose of <b>right hand</b>. See the \"OpenPose keypoints convention\" for guidance.</p>""",
-                    visible=False
-                )
-                # fix_kp_r_info = gr.Markdown(
-                    # """<p style="text-align: center; font-size: 20px; font-weight: bold; ">Select right only</p>""",
-                    # visible=False,
-                # )
-                fix_kp_right = gr.Image(
+                fix_kp_all = gr.Image(
                     type="numpy",
-                    label="Keypoint Selection (right hand)",
-                    show_label=True,
+                    # label="Keypoints",
+                    show_label=False,
                     height=LENGTH,
                     width=LENGTH,
                     interactive=False,
-                    visible=False,
-                    sources=[],
+                    visible=True,
+                    sources=(),
+                    image_mode="RGBA"
                 )
-                with gr.Row():
-                    fix_undo_right = gr.Button(
-                        value="Undo", interactive=False, visible=False
+                with gr.Accordion(open=True):
+                    fix_ex_kpts = gr.Examples(
+                        fix_example_kpts,
+                        inputs=[fix_kp_all, fix_kpts_path],
+                        examples_per_page=20,
+                        postprocess=False,
+                        elem_id="kpts_examples"
                     )
-                    fix_reset_right = gr.Button(
-                        value="Reset", interactive=False, visible=False
+                with gr.Accordion("[Custom data] Manually give hand pose", open=False):
+                    gr.Markdown(
+                        """<p style="text-align: center;">&#9312; Tell us if this is right, left, or both hands</p>"""
                     )
-                fix_kp_l_info = gr.Markdown(
-                    """<p style="text-align: center;">&#9313; Click 21 keypoints on the image to provide the target hand pose of <b>left hand</b>. See the \"OpenPose keypoints convention\" for guidance.</p>""",
-                    visible=False
-                )
-                fix_kp_left = gr.Image(
-                    type="numpy",
-                    label="Keypoint Selection (left hand)",
-                    show_label=True,
-                    height=LENGTH,
-                    width=LENGTH,
-                    interactive=False,
-                    visible=False,
-                    sources=[],
-                )
-                with gr.Row():
-                    fix_undo_left = gr.Button(
-                        value="Undo", interactive=False, visible=False
+                    fix_checkbox = gr.CheckboxGroup(
+                        ["Right hand", "Left hand"],
+                        show_label=False,
+                        interactive=False,
                     )
-                    fix_reset_left = gr.Button(
-                        value="Reset", interactive=False, visible=False
+                    fix_kp_r_info = gr.Markdown(
+                        """<p style="text-align: center;">&#9313; Click 21 keypoints on the image to provide the target hand pose of <b>right hand</b>. See the \"OpenPose keypoints convention\" for guidance.</p>""",
+                        visible=False
+                    )
+                    # fix_kp_r_info = gr.Markdown(
+                        # """<p style="text-align: center; font-size: 20px; font-weight: bold; ">Select right only</p>""",
+                        # visible=False,
+                    # )
+                    fix_kp_right = gr.Image(
+                        type="numpy",
+                        label="Keypoint Selection (right hand)",
+                        show_label=True,
+                        height=LENGTH,
+                        width=LENGTH,
+                        interactive=False,
+                        visible=False,
+                        sources=[],
+                    )
+                    with gr.Row():
+                        fix_undo_right = gr.Button(
+                            value="Undo", interactive=False, visible=False
+                        )
+                        fix_reset_right = gr.Button(
+                            value="Reset", interactive=False, visible=False
+                        )
+                    fix_kp_l_info = gr.Markdown(
+                        """<p style="text-align: center;">&#9313; Click 21 keypoints on the image to provide the target hand pose of <b>left hand</b>. See the \"OpenPose keypoints convention\" for guidance.</p>""",
+                        visible=False
+                    )
+                    fix_kp_left = gr.Image(
+                        type="numpy",
+                        label="Keypoint Selection (left hand)",
+                        show_label=True,
+                        height=LENGTH,
+                        width=LENGTH,
+                        interactive=False,
+                        visible=False,
+                        sources=[],
+                    )
+                    with gr.Row():
+                        fix_undo_left = gr.Button(
+                            value="Undo", interactive=False, visible=False
+                        )
+                        fix_reset_left = gr.Button(
+                            value="Reset", interactive=False, visible=False
+                        )
+                    gr.Markdown(
+                        """<p style="text-align: left; font-weight: bold; ">OpenPose keypoints convention</p>"""
+                    )
+                    fix_openpose = gr.Image(
+                        value="openpose.png",
+                        type="numpy",
+                        show_label=False,
+                        height=LENGTH // 2,
+                        width=LENGTH // 2,
+                        interactive=False,
                     )
-                gr.Markdown(
-                    """<p style="text-align: left; font-weight: bold; ">OpenPose keypoints convention</p>"""
-                )
-                fix_openpose = gr.Image(
-                    value="openpose.png",
-                    type="numpy",
-                    show_label=False,
-                    height=LENGTH // 2,
-                    width=LENGTH // 2,
-                    interactive=False,
-                )
             
             # get latent
             # with gr.Column():
@@ -1650,9 +1696,10 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         # fix_vis_mask32.change(
         #     enable_component, [fix_vis_mask32, fix_vis_mask256], fix_run
         # )
+        fix_kpts_path.change(read_kpts, fix_kpts_path, fix_kpts_np)
         fix_run.click(
             ready_sample,
-            [fix_ref, fix_inpaint_mask, fix_kpts],
+            [fix_ref, fix_inpaint_mask, fix_kpts, fix_kpts_np],
             [
                 fix_ref_cond,
                 fix_target_cond,
@@ -1663,7 +1710,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 fix_vis_mask256,
             ],
         )
-        fix_kpts_np.change(
+        fix_inpaint_latent.change(
             sample_inpaint,
             [
                 fix_ref_cond,
@@ -1688,6 +1735,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 fix_crop_coord,
                 fix_ref,
                 fix_checkbox,
+                fix_kp_all,
                 fix_kp_right,
                 fix_kp_left,
                 fix_result,
@@ -1704,6 +1752,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 fix_target_cond,
                 fix_latent,
                 fix_inpaint_latent,
+                fix_kpts_path,
                 fix_n_generation,
                 fix_seed,
                 fix_cfg,
@@ -2272,7 +2321,8 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
     gr.Markdown("<h1>Acknowledgement</h1>")
     gr.Markdown(_ACK_)
     gr.Markdown("<h1>Trouble Shooting</h1>")
-    gr.Markdown("If something doesn't work, please try the following steps:<br>1. Refresh the page and try again.<br>2. If the issue persists, kindly leave a message on our HuggingFace Spaces Community tab (located at the top right), on our GitHub repository's Issues page, or send us an email. We are happy to help you as soon as possible.<br>3. The issue might be due to compatibility with HuggingFace or GPU VRAM limitations. We recommend cloning this repository and trying it with your own GPU if possible.")
+    gr.Markdown("If error persists, please try the following steps:<br>1. Refresh the page and try again.<br>2. The issue might be due to compatibility with HuggingFace or GPU memory limitations. We recommend cloning this repository and trying it with your own GPU if possible.<br>3. Kindly leave a message on our HuggingFace Spaces Community tab (located at the top right), on our GitHub repository's Issues page, or send us an email. We are happy to help you as soon as possible.")
+    gr.Markdown("If the result is not satisfactory:<br>1. Try changing the Classifier Free Guidance Scale, that can be found at \"More Options\".")
     gr.Markdown("<h1>Citation</h1>")
     gr.Markdown(
         """<p style="text-align: left;">If this was useful, please cite us! ❤️</p>"""

bad_hands/1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65ec39b95455dcb42771a0fa21d087d9065d3cddd24b01d3200dbfb05a4108a9
+size 464

bad_hands/3.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876f8cc25adbd22e2a89f8bdeaa39bb519ff68b98f0831906dcdc5762e4fa0a2
+size 464

bad_hands/4.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57abffa5190455ebc915b16641e69fb491091ae626316a6d00bd3cb156696605
+size 800

bad_hands/5.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9ac0ad6f59679be44091daac9ee4e50e331f005d62845ba78788685de4b4d28
+size 800

bad_hands/6.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:950449c557f58ca5fa65ada7fcd0652a4f4781774c4ddd9e53aec1f9acaa1ca2
+size 800

bad_hands/7.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ae9e3f73fdc8908804e383a28817803c3e9bde63355e1327d2b4f1f24083d8b
+size 464

debug_keypts.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14b1e5273aded0649d0097c8032dff15900a071a6c0d82d9a775e74565c29cc9
+size 800

prepare_examples.py

@@ -0,0 +1,99 @@
+import glob
+import os
+from PIL import Image
+import numpy as np
+from matplotlib import pyplot as plt
+from io import BytesIO
+
+def visualize_hand(all_joints, img, side=["right", "left"], n_avail_joints=21):
+    # Define the connections between joints for drawing lines and their corresponding colors
+    connections = [
+        ((0, 1), "red"),
+        ((1, 2), "green"),
+        ((2, 3), "blue"),
+        ((3, 4), "purple"),
+        ((0, 5), "orange"),
+        ((5, 6), "pink"),
+        ((6, 7), "brown"),
+        ((7, 8), "cyan"),
+        ((0, 9), "yellow"),
+        ((9, 10), "magenta"),
+        ((10, 11), "lime"),
+        ((11, 12), "indigo"),
+        ((0, 13), "olive"),
+        ((13, 14), "teal"),
+        ((14, 15), "navy"),
+        ((15, 16), "gray"),
+        ((0, 17), "lavender"),
+        ((17, 18), "silver"),
+        ((18, 19), "maroon"),
+        ((19, 20), "fuchsia"),
+    ]
+    H, W, C = img.shape
+
+    # Create a figure and axis
+    plt.figure()
+    ax = plt.gca()
+    # Plot joints as points
+    ax.imshow(img)
+    start_is = []
+    if "right" in side:
+        start_is.append(0)
+    if "left" in side:
+        start_is.append(21)
+    for start_i in start_is:
+        joints = all_joints[start_i : start_i + n_avail_joints]
+        if len(joints) == 1:
+            ax.scatter(joints[0][0], joints[0][1], color="red", s=10)
+        else:
+            for connection, color in connections[: len(joints) - 1]:
+                joint1 = joints[connection[0]]
+                joint2 = joints[connection[1]]
+                ax.plot([joint1[0], joint2[0]], [joint1[1], joint2[1]], color=color, linewidth=4)
+
+    ax.set_xlim([0, W])
+    ax.set_ylim([0, H])
+    ax.grid(False)
+    ax.set_axis_off()
+    ax.invert_yaxis()
+    # plt.subplots_adjust(wspace=0.01)
+    # plt.show()
+    buf = BytesIO()
+    plt.savefig(buf, format="png", bbox_inches="tight", pad_inches=0)
+    plt.close()
+
+    # Convert BytesIO object to numpy array
+    buf.seek(0)
+    img_pil = Image.open(buf)
+    img_pil = img_pil.resize((W, H))
+    numpy_img = np.array(img_pil)
+
+    return numpy_img
+
+'''put brush example at alpha channel'''
+# img_dir = "bad_hands"
+# masked_paths = sorted(glob.glob(os.path.join(img_dir, "*_mask.jpg")))
+# for masked_pth in masked_paths:
+    # img_path = masked_pth.replace("_mask.jpg", ".jpg")
+    # assert os.path.exists(img_path), f"Image path {img_path} does not exist."
+    # masked = np.array(Image.open(masked_pth))
+    # mask = (np.all(masked > 245, axis=-1)).astype(np.uint8)*128 + 64
+    # img = np.array(Image.open(img_path))
+    # composite = np.concatenate((img, mask[..., None]), axis=-1)
+    # composite = Image.fromarray(composite)
+    # composite.save(masked_pth.replace("_mask.jpg", "_composite.png"))
+    # print(f"Saved composite image {masked_pth.replace('_mask.jpg', '_composite.png')}")
+
+'''visualize keypoint example'''
+data_dir = "bad_hands"
+kpts_paths = sorted(glob.glob(os.path.join(data_dir, "*.npy")))
+for kpts_pth in kpts_paths:
+    img_pth = kpts_pth.replace(".npy", ".jpg")
+    kpts = np.load(kpts_pth)
+    img = np.array(Image.open(img_pth))
+    h, w = img.shape[:2]
+    kpts = kpts / np.array([256,256]) * np.array([w, h])
+    kpts_vis = visualize_hand(kpts, img)
+    save_path = kpts_pth.replace(".npy", "_kpts.png")
+    kpts_vis = Image.fromarray(kpts_vis).save(save_path)
+    print(f"Saved {save_path}")

tmp.py

@@ -1,18 +0,0 @@
-import glob
-import os
-from PIL import Image
-import numpy as np
-
-img_dir = "bad_hands"
-masked_paths = sorted(glob.glob(os.path.join(img_dir, "*_mask.jpg")))
-for masked_pth in masked_paths:
-    img_path = masked_pth.replace("_mask.jpg", ".jpg")
-    assert os.path.exists(img_path), f"Image path {img_path} does not exist."
-    masked = np.array(Image.open(masked_pth))
-    mask = (np.all(masked > 245, axis=-1)).astype(np.uint8)*128 + 64
-    img = np.array(Image.open(img_path))
-    composite = np.concatenate((img, mask[..., None]), axis=-1)
-    # img.putalpha(Image.fromarray(mask))
-    composite = Image.fromarray(composite)
-    composite.save(masked_pth.replace("_mask.jpg", "_composite.png"))
-    print(f"Saved composite image {masked_pth.replace('_mask.jpg', '_composite.png')}")