use flux canny lora

app.py

@@ -15,8 +15,12 @@ subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
 # Load FLUX image generator
 device = "cuda" if torch.cuda.is_available() else "cpu"
 model_repo_id = "black-forest-labs/FLUX.1-schnell"  # Replace to the model you would like to use
-lora_path = "matteomarjanovic/flatsketcher"
-weigths_file = "lora.safetensors"
+flat_lora_path = "matteomarjanovic/flatsketcher"
+canny_lora_path = "black-forest-labs/FLUX.1-Canny-dev-lora"
+flat_weigths_file = "lora.safetensors"
+canny_weigths_file = "flux1-canny-dev-lora.safetensors"
+
+processor = CannyDetector()
 
 if torch.cuda.is_available():
     torch_dtype = torch.float16
@@ -25,7 +29,10 @@ else:
 
 pipe = DiffusionPipeline.from_pretrained(model_repo_id, torch_dtype=torch_dtype)
 pipe = pipe.to(device)
-pipe.load_lora_weights(lora_path, weight_name=weigths_file)
+pipe.load_lora_weights(flat_lora_path, weight_name=flat_weigths_file, adapter_name="flat")
+pipe.load_lora_weights(canny_lora_path, weight_name=cannyweigths_file, adapter_name="canny")
+
+pipe.set_adapters(["flat", "canny"], adapter_weights=[0.7, 0.7])
 
 MAX_SEED = np.iinfo(np.int32).max
 MAX_IMAGE_SIZE = 1024
@@ -94,15 +101,22 @@ def generate_description_fn(
     )
 
     prompt = chat_completion.choices[0].message.content + " In the style of FLTSKC"
+    control_image = processor(
+        image,
+        low_threshold=50,
+        high_threshold=200,
+        detect_resolution=1024,
+        image_resolution=1024
+    )
 
     image = pipe(
         prompt=prompt,
+        control_image=control_image
         guidance_scale=0.,
         num_inference_steps=4,
         width=1420,
         height=1080,
         max_sequence_length=256,
-        
     ).images[0]
 
     return prompt, image

requirements.txt

@@ -6,4 +6,5 @@ transformers
 xformers
 sentencepiece
 peft
-groq
+groq
+controlnet-aux