Spaces:

Dovakiins
/

qwerrwe

Build error

Maxime commited on Aug 26, 2023

Commit

a184549

unverified ·

1 Parent(s): f311df9

ignore: linter

Files changed (1) hide show

src/axolotl/utils/models.py CHANGED Viewed

@@ -368,7 +368,7 @@ def load_model(
     # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
     # convert them back to fp16/bf16 for flash-attn compatibility.
-    if (fix_dtype or cfg.adapter == "" or cfg.adapter == None) and (
         cfg.flash_attention and cfg.is_llama_derived_model
     ):
         for name, module in model.named_modules():

     # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
     # convert them back to fp16/bf16 for flash-attn compatibility.
+    if (fix_dtype or cfg.adapter == "" or cfg.adapter is None) and (
         cfg.flash_attention and cfg.is_llama_derived_model
     ):
         for name, module in model.named_modules():