Removed multiplicative / inverse stuff

app.py

@@ -116,12 +116,12 @@ def download_and_upload_non_model_files(base_model_name, output_repo_name):
         shutil.rmtree(temp_config_dir, ignore_errors=True)
 
 def merge_lora_efficient(hf_token, base_model_name, lora_model_name, output_repo_name,
-                         scale_factor, multiplicative_lora, inverse_lora, progress=gr.Progress()):
+                         scale_factor, progress=gr.Progress()):
     temp_lora_dir = None
     try:
         # Validate scale factor
-        if not (0 < scale_factor < 2):
-            error_msg = "Scale factor must be in the range (0, 2)"
+        if not (-2 <= scale_factor <= 2):
+            error_msg = "Scale factor must be in the range [-2, 2]"
             warning_fn(error_msg)
             return f"✗ Error: {error_msg}"
 
@@ -165,18 +165,6 @@ def merge_lora_efficient(hf_token, base_model_name, lora_model_name, output_repo
 
         info_fn(f"Found {len(shard_files)} model shards to process")
 
-        # Determine merge mode
-        if multiplicative_lora and inverse_lora:
-            merge_mode = "Multiplicative Inverse"
-        elif multiplicative_lora:
-            merge_mode = "Multiplicative"
-        elif inverse_lora:
-            merge_mode = "Additive Inverse"
-        else:
-            merge_mode = "Additive"
-        
-        info_fn(f"Merge mode: {merge_mode}")
-
         merged_tensors = 0
         total_shards = len(shard_files)
 
@@ -214,7 +202,7 @@ def merge_lora_efficient(hf_token, base_model_name, lora_model_name, output_repo
                         lora_A, lora_B = find_lora_weights(lora_state, key)
 
                         if lora_A is not None and lora_B is not None:
-                            info_fn(f"Merging {merge_mode} LoRA weights for {key}")
+                            info_fn(f"Merging LoRA weights for {key}")
                             shard_merged_count += 1
                             merged_tensors += 1
 
@@ -223,32 +211,11 @@ def merge_lora_efficient(hf_token, base_model_name, lora_model_name, output_repo
                             tensor = tensor.to(torch.float32)
                             lora_delta = scale * lora_B.to(torch.float32) @ lora_A.to(torch.float32)
 
-                            if multiplicative_lora:
-                                # Validate dimensions for multiplicative LoRA
-                                if lora_delta.shape[0] != lora_delta.shape[1]:
-                                    raise ValueError(f"Multiplicative LoRA requires square delta matrix for {key}: got shape {lora_delta.shape}")
-                                if lora_delta.shape[-1] != tensor.shape[-2]:
-                                    raise ValueError(f"Multiplicative LoRA dimension mismatch for {key}: {lora_delta.shape} vs {tensor.shape}")
-                                
-                                if inverse_lora:
-                                    # Inverse multiplicative: tensor = (I + lora_delta)^(-1) @ tensor
-                                    identity = torch.eye(lora_delta.shape[0], device=lora_delta.device, dtype=torch.float32)
-                                    inverse_matrix = torch.linalg.inv(identity + lora_delta)
-                                    tensor = inverse_matrix @ tensor
-                                else:
-                                    # Forward multiplicative: tensor = (I + lora_delta) @ tensor
-                                    tensor += lora_delta @ tensor
-                            else:
-                                # Validate dimensions for additive LoRA
-                                if lora_delta.shape != tensor.shape:
-                                    raise ValueError(f"Additive LoRA dimension mismatch for {key}: {lora_delta.shape} vs {tensor.shape}")
-                                
-                                if inverse_lora:
-                                    # Inverse additive: tensor = tensor - lora_delta
-                                    tensor -= lora_delta
-                                else:
-                                    # Forward additive: tensor = tensor + lora_delta
-                                    tensor += lora_delta
+                            # Validate dimensions for additive LoRA
+                            if lora_delta.shape != tensor.shape:
+                                raise ValueError(f"Additive LoRA dimension mismatch for {key}: {lora_delta.shape} vs {tensor.shape}")
+                            
+                            tensor += lora_delta
 
                             # Convert back to original dtype
                             tensor = tensor.to(original_dtype)
@@ -284,7 +251,7 @@ def merge_lora_efficient(hf_token, base_model_name, lora_model_name, output_repo
 
         progress(1.0, desc="Upload completed!")
 
-        success_msg = f"✓ Successfully merged and uploaded model!\nModel URL: https://huggingface.co/{output_repo_name}\nMerge mode: {merge_mode}\nScale factor: {scale_factor}\nProcessed {total_shards} shards\nMerged {merged_tensors} layers with LoRA weights"
+        success_msg = f"��� Successfully merged and uploaded model!\nModel URL: https://huggingface.co/{output_repo_name}\nScale factor: {scale_factor}\nProcessed {total_shards} shards\nMerged {merged_tensors} layers with LoRA weights"
         info_fn("Merge completed successfully!")
 
         return success_msg
@@ -310,23 +277,21 @@ This tool merges LoRA (Low-Rank Adaptation) adapters with base models using a me
 - **Streaming Processing**: Downloads → Processes → Uploads → Deletes each shard sequentially
 - **Automatic Cleanup**: Temporary files are automatically removed after processing
 - **Progress Tracking**: Real-time status updates throughout the merge process
-- **Advanced Options**: Multiplicative LoRA, inverse merging, and custom scale factors
+- **Advanced Options**: Custom scale factors (including negative values)
 """
 
 DETAILS_TEXT = """
 ### How It Works
-LoRA enables efficient fine-tuning by adding small adapter weights rather than modifying the entire model. This tool supports four merge modes:
+LoRA enables efficient fine-tuning by adding small adapter weights rather than modifying the entire model. This tool applies additive merging:
 
 - **Additive LoRA**: `W_new = W + scale × B @ A`
-- **Additive Inverse**: `W_new = W - scale × B @ A` (removes LoRA effect)
-- **Multiplicative LoRA**: `W_new = W + scale × B @ A @ W`
-- **Multiplicative Inverse**: `W_new = (I + scale × B @ A)^(-1) @ W`
+- Negative scale values reverse the effect
 
 ### Scale Factor
-The scale factor (0 < scale < 2) controls the strength of the LoRA merge:
+The scale factor (-2 ≤ scale ≤ 2) controls the strength of the LoRA merge:
 - **1.0**: Full strength (default)
 - **0.5**: Half strength
-- **1.5**: 150% strength
+- **-1.0**: Reverse effect (removes LoRA impact)
 
 ### Memory Efficiency
 - **Traditional approach**: Loads entire model (~15GB+ for 7B parameter models)
@@ -375,22 +340,12 @@ with gr.Blocks(title="Memory-Efficient LoRA Merge", theme=gr.themes.Soft()) as d
 
             gr.Markdown("### Advanced Options")
             scale_factor = gr.Slider(
-                minimum=0.01,
-                maximum=1.99,
+                minimum=-2.0,
+                maximum=2.0,
                 value=1.0,
                 step=0.01,
                 label="Scale Factor",
-                info="Strength of LoRA merge (0 < scale < 2)"
-            )
-            multiplicative_lora = gr.Checkbox(
-                label="Multiplicative LoRA",
-                value=False,
-                info="Apply multiplicative LoRA instead of additive LoRA"
-            )
-            inverse_lora = gr.Checkbox(
-                label="Inverse Merge",
-                value=False,
-                info="Apply inverse operation (subtract/invert the LoRA effect)"
+                info="Strength of LoRA merge (-2 ≤ scale ≤ 2)"
             )
 
         with gr.Column(scale=1):
@@ -408,7 +363,7 @@ with gr.Blocks(title="Memory-Efficient LoRA Merge", theme=gr.themes.Soft()) as d
     submit_btn.click(
         fn=merge_lora_efficient,
         inputs=[hf_token, base_model_name, lora_model_name, output_repo_name, 
-                scale_factor, multiplicative_lora, inverse_lora],
+                scale_factor],
         outputs=output_text
     )