hyper sdxl

app.py

@@ -5,17 +5,11 @@ import tempfile
 import numpy as np
 from pathlib import Path
 from PIL import Image
-from diffusers import (
-    ControlNetModel,
-    StableDiffusionXLControlNetPipeline,
-    UNet2DConditionModel,
-    EulerDiscreteScheduler,
-)
+from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, TCDScheduler
 import spaces
 import gradio as gr
 from huggingface_hub import hf_hub_download, snapshot_download
 from ip_adapter import IPAdapterXL
-from safetensors.torch import load_file
 
 snapshot_download(
     repo_id="h94/IP-Adapter", allow_patterns="sdxl_models/*", local_dir="."
@@ -36,7 +30,7 @@ controlnet = ControlNetModel.from_pretrained(
     controlnet_path, use_safetensors=False, torch_dtype=torch.float16
 ).to(device)
 
-# load SDXL lightnining
+# load Hyper SD
 
 pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
     base_model_path,
@@ -46,17 +40,11 @@ pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
     add_watermarker=False,
 ).to(device)
 pipe.set_progress_bar_config(disable=True)
-pipe.scheduler = EulerDiscreteScheduler.from_config(
-    pipe.scheduler.config, timestep_spacing="trailing", prediction_type="epsilon"
-)
-pipe.unet.load_state_dict(
-    load_file(
-        hf_hub_download(
-            "ByteDance/SDXL-Lightning", "sdxl_lightning_2step_unet.safetensors"
-        ),
-        device="cuda",
-    )
+pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
+pipe.load_lora_weights(
+    hf_hub_download("ByteDance/Hyper-SD", "Hyper-SDXL-1step-lora.safetensors")
 )
+eta = 1.0
 
 # load ip-adapter
 # target_blocks=["block"] for original IP-Adapter
@@ -225,6 +213,7 @@ def create_image(
             controlnet_conditioning_scale=float(control_scale),
             neg_content_prompt=neg_content_prompt,
             neg_content_scale=neg_content_scale,
+            eta=1.0,
         )
     else:
         images = ip_model.generate(
@@ -238,6 +227,7 @@ def create_image(
             seed=seed,
             image=canny_map,
             controlnet_conditioning_scale=float(control_scale),
+            eta=1.0,
         )
     image = images[0]
     with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmpfile:
@@ -258,7 +248,7 @@ title = r"""
 
 description = r"""
 <b>Forked from <a href='https://github.com/InstantStyle/InstantStyle' target='_blank'>InstantStyle: Free Lunch towards Style-Preserving in Text-to-Image Generation</a>.<br>
-<b>Model by <a href='https://huggingface.co/ByteDance/SDXL-Lightning' target='_blank'>SDXL Lightning</a> and <a href='https://huggingface.co/h94/IP-Adapter' target='_blank'>IP-Adapter</a>.</b><br>
+<b>Model by <a href='https://huggingface.co/ByteDance/Hyper-SD' target='_blank'>Hyper-SD</a> and <a href='https://huggingface.co/h94/IP-Adapter' target='_blank'>IP-Adapter</a>.</b><br>
 """
 
 article = r"""
@@ -347,10 +337,10 @@ with block:
                         label="guidance scale",
                     )
                     num_inference_steps = gr.Slider(
-                        minimum=2,
-                        maximum=50.0,
+                        minimum=1,
+                        maximum=10.0,
                         step=1.0,
-                        value=2,
+                        value=1,
                         label="num inference steps",
                     )
                     seed = gr.Slider(
@@ -390,6 +380,10 @@ with block:
             scale.input,
             control_scale.input,
             seed.input,
+            num_inference_steps.input,
+            target.input,
+            neg_content_prompt.input,
+            neg_content_scale.input,
         ],
         fn=create_image,
         inputs=inputs,