final touch/test, tutorials

.gitignore

@@ -6,4 +6,5 @@ diffusion/__pycache__/
 *.pyc
 DINO_EMA_11M_b50_lr1e-5_epoch6_step320k.ckpt
 sam_vit_h_4b8939.pth
-vae-ft-mse-840000-ema-pruned.ckpt
+vae-ft-mse-840000-ema-pruned.ckpt
+how_to_videos/

app.py

@@ -17,8 +17,6 @@ from io import BytesIO
 from PIL import Image
 import random
 from copy import deepcopy
-from typing import Optional
-import requests
 from huggingface_hub import hf_hub_download
 try:
     import spaces
@@ -47,10 +45,7 @@ def set_seed(seed):
     torch.cuda.manual_seed_all(seed)
     random.seed(seed)
 
-# if torch.cuda.is_available():
 device = "cuda"
-# else:
-    # device = "cpu"
 
 def remove_prefix(text, prefix):
     if text.startswith(prefix):
@@ -487,16 +482,6 @@ def visualize_ref(ref):
     img = mask_image(img, mask)
     return img, inpaint_mask
 
-# def inpaint_from_example(ex_mask):
-#     inpaint_mask = (ex_mask[:,:, -1] == 0).astype(np.uint8)
-#     inpaint_mask = np.tile(inpaint_mask[..., None], (1, 1, 4))
-#     ret = {
-#         "background": cv2.cvtColor(ex_mask, cv2.COLOR_RGB2RGBA),
-#         "layers": [inpaint_mask],
-#         "composite": cv2.cvtColor(ex_mask, cv2.COLOR_RGB2RGBA)
-#     }
-#     return ret
-
 def get_kps(img, keypoints, side: Literal["right", "left"], evt: gr.SelectData):
     if keypoints is None:
         keypoints = [[], []]
@@ -630,7 +615,8 @@ def sample_diff(ref_cond, target_cond, target_keypts, num_gen, seed, cfg):
         nvs=torch.cat([nvs, 2 * torch.ones_like(nvs)]),
         cfg_scale=cfg,
     )
-
+    
+    gr.Info("The process successfully started to run. Please wait for 50s x Number of Generation.", duration=20)
     samples, _ = diffusion.p_sample_loop(
         model.forward_with_cfg,
         z.shape,
@@ -835,7 +821,7 @@ def sample_inpaint(
         cfg_scale=cfg_scale,
     )
 
-    gr.Info("The process successfully started to run. Please wait for around 3.5 minutes.", duration=20)
+    gr.Info("The process successfully started to run. Please wait for around 3.5 minutes.", duration=220)
     samples, _ = diffusion.inpaint_p_sample_loop(
         model.forward_with_cfg,
         z.shape,
@@ -1224,17 +1210,15 @@ fix_example_brush = [
     # ["bad_hands/15_mask.jpg"],
 ]
 fix_example_kpts = [
-    ["bad_hands/1_kpts.png"],
-    ["bad_hands/3_kpts.png"],
-    ["bad_hands/4_kpts.png"],
-    ["bad_hands/5_kpts.png"],
-    ["bad_hands/6_kpts.png"],
-    ["bad_hands/7_kpts.png"],
+    ["bad_hands/1_kpts.png", 3.0, 1224],
+    ["bad_hands/3_kpts.png", 1.0, 42],
+    ["bad_hands/4_kpts.png", 2.0, 42],
+    ["bad_hands/5_kpts.png", 3.0, 42],
+    ["bad_hands/6_kpts.png", 3.0, 1348],
+    ["bad_hands/7_kpts.png", 3.0, 42],
 ]
 for i in range(len(fix_example_kpts)):
     npy_path = fix_example_kpts[i][0].replace("_kpts.png", ".npy")
-    # kpts = np.load(npy_path)
-    # kpts = gr.State(kpts)
     fix_example_kpts[i].append(npy_path)
 
 custom_css = """
@@ -1256,8 +1240,21 @@ custom_css = """
 #kpts_examples table tr td:nth-child(2) {
     display: none !important;
 }
+#kpts_examples table tr th:nth-child(3),
+#kpts_examples table tr td:nth-child(3) {
+    display: none !important;
+}
+#kpts_examples table tr th:nth-child(4),
+#kpts_examples table tr td:nth-child(4) {
+    display: none !important;
+}
+#repose_tutorial video {
+    width: 70% !important;
+    display: block;
+    margin: 0 auto;
+    padding: 0;
+}
 """
-#    color: black !important;
 
 _HEADER_ = '''
 <div style="text-align: center;">
@@ -1309,7 +1306,6 @@ Part of this work was done during Kefan (Arthur) Chen’s internship at Meta Rea
 with gr.Blocks(css=custom_css, theme="soft") as demo:
     gr.Markdown(_HEADER_)
     with gr.Tab("Demo 1. Malformed Hand Correction", elem_id="fix-tab"):
-        # gr.Markdown("""<p style="background-color: #FFDAB9; text-align: center; font-size: 20px; font-weight: bold;">Demo 1. Malformed Hand Correction</p>""")
         fix_inpaint_mask = gr.State(value=None)
         fix_original = gr.State(value=None)
         fix_crop_coord = gr.State(value=None)
@@ -1321,6 +1317,72 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         fix_target_cond = gr.State(value=None)
         fix_latent = gr.State(value=None)
         fix_inpaint_latent = gr.State(value=None)
+
+        # tutorial video
+        with gr.Accordion():
+            gr.Markdown("""<p style="text-align: center; font-size: 20px; font-weight: bold;">Tutorial Videos of Demo 1</p>""")
+            with gr.Row(variant="panel"):
+                with gr.Column():
+                    gr.Video(
+                        "how_to_videos/subtitled_fix_hands_custom.mp4",
+                        label="Using your own image",
+                        autoplay=True,
+                        loop=True,
+                        show_label=True,
+                    )
+                with gr.Column():
+                    gr.Video(
+                        "how_to_videos/subtitled_fix_hands_example.mp4",
+                        label="Using our example image",
+                        autoplay=True,
+                        loop=True,
+                        show_label=True,
+                    )
+        
+        # more options
+        with gr.Accordion(label="More options", open=False):
+            gr.Markdown(
+                "⚠️ Currently, Number of generation > 1 could lead to out-of-memory"
+            )
+            with gr.Row():
+                fix_n_generation = gr.Slider(
+                    label="Number of generations",
+                    value=1,
+                    minimum=1,
+                    maximum=FIX_MAX_N,
+                    step=1,
+                    randomize=False,
+                    interactive=True,
+                )
+                fix_seed = gr.Slider(
+                    label="Seed",
+                    value=42,
+                    minimum=0,
+                    maximum=10000,
+                    step=1,
+                    randomize=False,
+                    interactive=True,
+                )
+                fix_cfg = gr.Slider(
+                    label="Classifier free guidance scale",
+                    value=3.0,
+                    minimum=0.0,
+                    maximum=10.0,
+                    step=0.1,
+                    randomize=False,
+                    interactive=True,
+                )
+                fix_quality = gr.Slider(
+                    label="Quality",
+                    value=10,
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    randomize=False,
+                    interactive=True,
+                )
+        
+        # main tabs
         with gr.Row():
             # crop & brush
             with gr.Column():
@@ -1418,6 +1480,9 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 gr.Markdown(
                     """<p style="text-align: center; font-size: 18px; font-weight: bold;">3. Target hand pose</p>"""
                 )
+                gr.Markdown(
+                    """<p style="text-align: center;">Either get hand pose from Examples, or manually give hand pose (located at the bottom)</p>"""
+                )
                 fix_kp_all = gr.Image(
                     type="numpy",
                     # label="Keypoints",
@@ -1432,7 +1497,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 with gr.Accordion(open=True):
                     fix_ex_kpts = gr.Examples(
                         fix_example_kpts,
-                        inputs=[fix_kp_all, fix_kpts_path],
+                        inputs=[fix_kp_all, fix_cfg, fix_seed, fix_kpts_path],
                         examples_per_page=20,
                         postprocess=False,
                         elem_id="kpts_examples"
@@ -1580,58 +1645,12 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 )
                 fix_clear = gr.ClearButton()
 
-        with gr.Accordion(label="More options", open=False):
-            gr.Markdown(
-                "⚠️ Currently, Number of generation > 1 could lead to out-of-memory"
-            )
-            with gr.Row():
-                fix_n_generation = gr.Slider(
-                    label="Number of generations",
-                    value=1,
-                    minimum=1,
-                    maximum=FIX_MAX_N,
-                    step=1,
-                    randomize=False,
-                    interactive=True,
-                )
-                fix_seed = gr.Slider(
-                    label="Seed",
-                    value=42,
-                    minimum=0,
-                    maximum=10000,
-                    step=1,
-                    randomize=False,
-                    interactive=True,
-                )
-                fix_cfg = gr.Slider(
-                    label="Classifier free guidance scale",
-                    value=3.0,
-                    minimum=0.0,
-                    maximum=10.0,
-                    step=0.1,
-                    randomize=False,
-                    interactive=True,
-                )
-                fix_quality = gr.Slider(
-                    label="Quality",
-                    value=10,
-                    minimum=1,
-                    maximum=10,
-                    step=1,
-                    randomize=False,
-                    interactive=True,
-                )
         
         # listeners
-        # fix_crop.change(resize_to_full, fix_crop, fix_ref)
         fix_crop.change(stash_original, fix_crop, fix_original)  # fix_original: (real_H, real_W, 3)
         fix_crop.change(stay_crop, [fix_crop, fix_crop_coord], [fix_crop_coord, fix_ref])
         fix_crop.select(process_crop, [fix_crop, fix_crop_coord], [fix_crop_coord, fix_ref, fix_crop])
-        # fix_ref.change(disable_crop, fix_crop_coord, fix_crop)
-        # fix_ref.change(enable_component, [fix_crop, fix_crop], fix_ref)
-        # fix_ref.change(enable_component, [fix_crop, fix_crop], fix_finish_crop)
         fix_ref.change(visualize_ref, [fix_ref], [fix_img, fix_inpaint_mask])
-        # fix_finish_crop.click(get_mask_inpaint, [fix_ref], [])  # fix_ref: (real_cropped_H, real_cropped_W, 3)
         fix_img.change(lambda x: x, [fix_img], [fix_kp_right])
         fix_img.change(lambda x: x, [fix_img], [fix_kp_left])
         fix_ref.change(
@@ -1693,9 +1712,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         fix_reset_left.click(
             reset_kps, [fix_img, fix_kpts, gr.State("left")], [fix_kp_left, fix_kpts]
         )
-        # fix_vis_mask32.change(
-        #     enable_component, [fix_vis_mask32, fix_vis_mask256], fix_run
-        # )
         fix_kpts_path.change(read_kpts, fix_kpts_path, fix_kpts_np)
         fix_run.click(
             ready_sample,
@@ -1775,10 +1791,8 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         )
 
     with gr.Tab("Demo 2. Repose Hands", elem_id="repose-tab"):
-        # gr.Markdown("""<p style="background-color: #90EE90; text-align: center; font-size: 20px; font-weight: bold;">Demo 2. Repose Hands</p>""")
-        dump = gr.State(value=None)
-
         # ref states
+        dump = gr.State(value=None)
         ref_img = gr.State(value=None)
         ref_im_raw = gr.State(value=None)
         ref_kp_raw = gr.State(value=0)
@@ -1801,7 +1815,20 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         target_auto_cond = gr.State(value=None)
         target_cond = gr.State(value=None)
 
-        # main tab
+        # tutorial video
+        with gr.Accordion(""):
+            gr.Markdown("""<p style="text-align: center; font-size: 20px; font-weight: bold;">Tutorial Videos of Demo 2</p>""")
+            with gr.Row(variant="panel", elem_id="repose_tutorial"):
+                with gr.Column():
+                    gr.Video(
+                        "how_to_videos/subtitled_repose_hands.mp4",
+                        label="Tutorial",
+                        autoplay=True,
+                        loop=True,
+                        show_label=True,
+                    )
+
+        # main tabs
         with gr.Row():
             # ref column
             with gr.Column():
@@ -1822,10 +1849,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                     crop_size="1:1",
                 )
                 gr.Examples(example_ref_imgs, [ref], examples_per_page=20)
-                # gr.Markdown(
-                #     """<p style="text-align: center;">&#9313; Hit the &quot;Finish Cropping&quot; button to get hand pose</p>"""
-                # )
-                # ref_finish_crop = gr.Button(value="Finish Cropping", interactive=False)
                 with gr.Accordion(label="See hand pose and more options", open=False):
                     with gr.Tab("Automatic hand keypoints"):
                         ref_pose = gr.Image(
@@ -1944,12 +1967,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                     crop_size="1:1",
                 )
                 gr.Examples(example_target_imgs, [target], examples_per_page=20)
-                # gr.Markdown(
-                #     """<p style="text-align: center;">&#9313; Hit the &quot;Finish Cropping&quot; button to get hand pose</p>"""
-                # )
-                # target_finish_crop = gr.Button(
-                #     value="Finish Cropping", interactive=False
-                # )
                 with gr.Accordion(label="See hand pose and more options", open=False):
                     with gr.Tab("Automatic hand keypoints"):
                         target_pose = gr.Image(
@@ -2084,6 +2101,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 )
                 clear = gr.ClearButton()
 
+        # more options
         with gr.Accordion(label="More options", open=False):
             with gr.Row():
                 n_generation = gr.Slider(
@@ -2115,7 +2133,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 )
 
         # reference listeners
-        # ref.change(enable_component, [ref, ref], ref_finish_crop)
         ref.change(prepare_anno, [ref, ref_is_user], [ref_im_raw, ref_kp_raw])
         ref_kp_raw.change(lambda x: x, ref_im_raw, ref_manual_kp_right)
         ref_kp_raw.change(lambda x: x, ref_im_raw, ref_manual_kp_left)
@@ -2179,7 +2196,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         )
 
         # target listeners
-        # target.change(enable_component, [target, target], target_finish_crop)
         target.change(prepare_anno, [target, target_is_user], [target_im_raw, target_kp_raw])
         target_kp_raw.change(lambda x:x, target_im_raw, target_manual_kp_right)
         target_kp_raw.change(lambda x:x, target_im_raw, target_manual_kp_left)
@@ -2249,8 +2265,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
         # run listerners
         ref_cond.change(enable_component, [ref_cond, target_cond], run)
         target_cond.change(enable_component, [ref_cond, target_cond], run)
-        # ref_manual_pose.change(enable_component, [ref_manual_pose, target_manual_pose], run)
-        # target_manual_pose.change(enable_component, [ref_manual_pose, target_manual_pose], run)
         run.click(
             sample_diff,
             [ref_cond, target_cond, target_keypts, n_generation, seed, cfg],
@@ -2322,7 +2336,7 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
     gr.Markdown(_ACK_)
     gr.Markdown("<h1>Trouble Shooting</h1>")
     gr.Markdown("If error persists, please try the following steps:<br>1. Refresh the page and try again.<br>2. The issue might be due to compatibility with HuggingFace or GPU memory limitations. We recommend cloning this repository and trying it with your own GPU if possible.<br>3. Kindly leave a message on our HuggingFace Spaces Community tab (located at the top right), on our GitHub repository's Issues page, or send us an email. We are happy to help you as soon as possible.")
-    gr.Markdown("If the result is not satisfactory:<br>1. Try changing the Classifier Free Guidance Scale, that can be found at \"More Options\".")
+    gr.Markdown("If the result is not satisfactory:<br>1. Try changing either <b>Classifier Free Guidance Scale</b> or <b>Seed</b>, that can be found at \"More Options\".")
     gr.Markdown("<h1>Citation</h1>")
     gr.Markdown(
         """<p style="text-align: left;">If this was useful, please cite us! ❤️</p>"""