Spaces:

Dovakiins
/

qwerrwe

Build error

Maxime

tmm1 commited on Aug 27, 2023

Commit

7fd662d

unverified ·

1 Parent(s): 9e69968

Update src/axolotl/utils/models.py

Co-authored-by: Aman Gupta Karmani <[email protected]>

Files changed (1) hide show

src/axolotl/utils/models.py CHANGED Viewed

@@ -368,7 +368,7 @@ def load_model(
     # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
     # convert them back to fp16/bf16 for flash-attn compatibility.
-    if (fix_dtype or not cfg.adapter) and (
         cfg.flash_attention and cfg.is_llama_derived_model
     ):
         for name, module in model.named_modules():

     # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
     # convert them back to fp16/bf16 for flash-attn compatibility.
+    if fix_dtype and (
         cfg.flash_attention and cfg.is_llama_derived_model
     ):
         for name, module in model.named_modules():