Update lora_w2w.py

lora_w2w.py

@@ -83,7 +83,7 @@ class LoRAModule(nn.Module):
             alpha = alpha.detach().numpy()
         alpha = lora_dim if alpha is None or alpha == 0 else alpha
         self.scale = alpha / self.lora_dim
-        #self.scale = self.scale.bfloat16()
+        self.scale = self.scale.bfloat16()
         
 
         self.multiplier = multiplier
@@ -96,6 +96,7 @@ class LoRAModule(nn.Module):
 
     def forward(self, x):
         print(self.org_forward(x).dtype)
+        print((x@(([email protected])*self.std1+self.mean1).T).dtype)
 
         return self.org_forward(x) +\
             (x@(([email protected])*self.std1+self.mean1).T)@((([email protected])*self.std2+self.mean2))*self.multiplier*self.scale