move lora loading into optim code to work on CUDA memory (#2)

- Update app.py (df235a97686d6b1c302987ff376d754e0fd88610)
- Update optimization.py (083c84cbba42fa8be522b4b7be04a880decce3d2)

app.py

@@ -57,22 +57,22 @@ pipe = WanImageToVideoPipeline.from_pretrained(MODEL_ID,
 # pipe.unload_lora_weights()
 
 
-pipe.load_lora_weights(
-   "vrgamedevgirl84/Wan14BT2VFusioniX", 
-   weight_name="FusionX_LoRa/Phantom_Wan_14B_FusionX_LoRA.safetensors", 
-    adapter_name="phantom"
-)
-kwargs = {}
-kwargs["load_into_transformer_2"] = True
-pipe.load_lora_weights(
-   "vrgamedevgirl84/Wan14BT2VFusioniX", 
-   weight_name="FusionX_LoRa/Phantom_Wan_14B_FusionX_LoRA.safetensors", 
-    adapter_name="phantom_2", **kwargs
-)
-pipe.set_adapters(["phantom", "phantom_2"], adapter_weights=[1., 1.])
-pipe.fuse_lora(adapter_names=["phantom"], lora_scale=3., components=["transformer"])
-pipe.fuse_lora(adapter_names=["phantom_2"], lora_scale=1., components=["transformer_2"])
-pipe.unload_lora_weights()
+# pipe.load_lora_weights(
+#    "vrgamedevgirl84/Wan14BT2VFusioniX", 
+#    weight_name="FusionX_LoRa/Phantom_Wan_14B_FusionX_LoRA.safetensors", 
+#     adapter_name="phantom"
+# )
+# kwargs = {}
+# kwargs["load_into_transformer_2"] = True
+# pipe.load_lora_weights(
+#    "vrgamedevgirl84/Wan14BT2VFusioniX", 
+#    weight_name="FusionX_LoRa/Phantom_Wan_14B_FusionX_LoRA.safetensors", 
+#     adapter_name="phantom_2", **kwargs
+# )
+# pipe.set_adapters(["phantom", "phantom_2"], adapter_weights=[1., 1.])
+# pipe.fuse_lora(adapter_names=["phantom"], lora_scale=3., components=["transformer"])
+# pipe.fuse_lora(adapter_names=["phantom_2"], lora_scale=1., components=["transformer_2"])
+# pipe.unload_lora_weights()
 
 for i in range(3): 
     gc.collect()

optimization.py

@@ -43,6 +43,23 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
     @spaces.GPU(duration=1500)
     def compile_transformer():
         
+        pipeline.load_lora_weights(
+           "vrgamedevgirl84/Wan14BT2VFusioniX", 
+           weight_name="FusionX_LoRa/Phantom_Wan_14B_FusionX_LoRA.safetensors", 
+            adapter_name="phantom"
+        )
+        kwargs = {}
+        kwargs["load_into_transformer_2"] = True
+        pipeline.load_lora_weights(
+           "vrgamedevgirl84/Wan14BT2VFusioniX", 
+           weight_name="FusionX_LoRa/Phantom_Wan_14B_FusionX_LoRA.safetensors", 
+            adapter_name="phantom_2", **kwargs
+        )
+        pipeline.set_adapters(["phantom", "phantom_2"], adapter_weights=[1., 1.])
+        pipeline.fuse_lora(adapter_names=["phantom"], lora_scale=3., components=["transformer"])
+        pipeline.fuse_lora(adapter_names=["phantom_2"], lora_scale=1., components=["transformer_2"])
+        pipeline.unload_lora_weights()
+        
         with capture_component_call(pipeline, 'transformer') as call:
             pipeline(*args, **kwargs)