Update app.py

app.py

@@ -313,13 +313,27 @@ def run(state, drawpad):
     seed_everything(state.seed if state.seed >=0 else np.random.randint(2147483647))
     print('Generate!')
 
-    background = drawpad['background'].convert('RGBA')
+    background = drawpad['background']
+    if background is None:
+        background = Image.new(size=(opt.width, opt.height), mode='RGB', color=(255, 255, 255))
+    else:
+        background = background.convert('RGBA')
     inpainting_mode = np.asarray(background).sum() != 0
+    if not inpainting_mode:
+        background = Image.new(size=(opt.width, opt.height), mode='RGB', color=(255, 255, 255))
+        background_prompt = "Simple white background"
+    else:
+        background_prompt = None
     print('Inpainting mode: ', inpainting_mode)
 
-    user_input = np.asarray(drawpad['layers'][0]) # (H, W, 4)
-    foreground_mask = torch.tensor(user_input[..., -1])[None, None] # (1, 1, H, W)
-    user_input = torch.tensor(user_input[..., :-1]) # (H, W, 3)
+    if drawpad['composite'] is None:
+        user_input = np.zeros((opt.height, opt.width, 4))
+        foreground_mask = torch.zeros((1, 1, opt.height, opt.width))
+        user_input = torch.tensor(user_input[..., :-1]) # (H, W, 3)
+    else:
+        user_input = np.asarray(drawpad['composite']) # (H, W, 4)
+        foreground_mask = torch.tensor(user_input[..., -1])[None, None] # (1, 1, H, W)
+        user_input = torch.tensor(user_input[..., :-1]) # (H, W, 3)
 
     palette = torch.tensor([
         tuple(int(s[i+1:i+3], 16) for i in (0, 2, 4))
@@ -603,7 +617,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
         state.seed = opt.seed
         return state
 
-    state = gr.State(value=_define_state)
+    state = gr.State(value=_define_state())
 
 
     ### Demo user interface
@@ -612,13 +626,13 @@ with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
         """
 <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
     <div>
-        <h1>🧠 Semantic Palette with <font class="rainbow rainbow_text_animated">Stable Diffusion 3</font> 🎨</h1>
+        <h1>🧠 Semantic Draw Canvas with <font class="rainbow rainbow_text_animated">Stable Diffusion 3</font> 🎨</h1>
         <h5 style="margin: 0;">powered by</h5>
-        <h3>StreamMultiDiffusion: Real-Time Interactive Generation with Region-Based Semantic Control</h3>
+        <h3>SemanticDraw: Towards Real-Time Interactive Content Creation from Image Diffusion Models</h3>
         <h5 style="margin: 0;">If you ❤️ our project, please visit our Github and give us a 🌟!</h5>
         </br>
         <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
-            <a href='https://jaerinlee.com/research/StreamMultiDiffusion'>
+            <a href='https://jaerinlee.com/research/semantic-draw'>
                 <img src='https://img.shields.io/badge/Project-Page-green' alt='Project Page'>
             </a>
             &nbsp;
@@ -626,32 +640,32 @@ with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
                 <img src="https://img.shields.io/badge/arXiv-2403.09055-red">
             </a>
             &nbsp;
-            <a href='https://github.com/ironjr/StreamMultiDiffusion'>
-                <img src='https://img.shields.io/github/stars/ironjr/StreamMultiDiffusion?label=Github&color=blue'>
+            <a href='https://github.com/ironjr/semantic-draw'>
+                <img src='https://img.shields.io/github/stars/ironjr/semantic-draw?label=Github&color=blue'>
             </a>
             &nbsp;
             <a href='https://twitter.com/_ironjr_'>
                 <img src='https://img.shields.io/twitter/url?label=_ironjr_&url=https%3A%2F%2Ftwitter.com%2F_ironjr_'>
             </a>
             &nbsp;
-            <a href='https://github.com/ironjr/StreamMultiDiffusion/blob/main/LICENSE'>
+            <a href='https://github.com/ironjr/semantic-draw/blob/main/LICENSE'>
                 <img src='https://img.shields.io/badge/license-MIT-lightgrey'>
             </a>
             &nbsp;
-            <a href='https://huggingface.co/spaces/ironjr/StreamMultiDiffusion'>
-                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-StreamMultiDiffusion-yellow'>
+            <a href='https://huggingface.co/spaces/ironjr/semantic-draw'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-semantic-draw-yellow'>
             </a>
             &nbsp;
-            <a href='https://huggingface.co/spaces/ironjr/SemanticPalette'>
-                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-SD1.5-yellow'>
+            <a href='https://huggingface.co/spaces/ironjr/semantic-draw-canvas-sd15'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-CanvasSD1.5-yellow'>
             </a>
             &nbsp;
-            <a href='https://huggingface.co/spaces/ironjr/SemanticPaletteXL'>
-                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-SDXL-yellow'>
+            <a href='https://huggingface.co/spaces/ironjr/semantic-draw-canvas-sdxl'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-CanvasSDXL-yellow'>
             </a>
             &nbsp;
-            <a href='https://huggingface.co/spaces/ironjr/SemanticPalette3'>
-                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-SD3-yellow'>
+            <a href='https://huggingface.co/spaces/ironjr/semantic-draw-canvas-sd3'>
+                <img src='https://img.shields.io/badge/%F0%9F%A4%97%20Demo-CanvasSD3-yellow'>
             </a>
         </div>
     </div>
@@ -787,11 +801,11 @@ with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
 </div>
 <div style="display: inline-block; margin-top: 10px">
     <div class="gallery">
-        <a href="https://huggingface.co/spaces/ironjr/SemanticPaletteXL" target="_blank">
-            <img alt="AnimagineXL3.1 Demo" src="https://github.com/ironjr/StreamMultiDiffusion/blob/main/demo/semantic_palette_sd3/examples/icons/sdxl.webp?raw=true">
+        <a href="https://huggingface.co/spaces/ironjr/semantic-draw-canvas-sdxl" target="_blank">
+            <img alt="AnimagineXL3.1 Demo" src="https://github.com/ironjr/semantic-draw/blob/main/demo/canvas_sd3/examples/icons/sdxl.webp?raw=true">
         </a>
-        <a href="https://huggingface.co/spaces/ironjr/StreamMultiDiffusion" target="_blank">
-            <img alt="StreamMultiDiffusion Demo" src="https://github.com/ironjr/StreamMultiDiffusion/blob/main/demo/semantic_palette_sd3/examples/icons/smd.gif?raw=true">
+        <a href="https://huggingface.co/spaces/ironjr/semantic-draw" target="_blank">
+            <img alt="SemanticDraw Demo" src="https://github.com/ironjr/semantic-draw/blob/main/demo/canvas_sd3/examples/icons/smd.gif?raw=true">
         </a>
     </div>
 </div>