Update src/adapters/methods/reft.py

Co-authored-by: Leon Engländer <leon.englaender@gmail.com>
adapter-hub · Jan 18, 2025 · e964973 · e964973
1 parent a66f4ae
commit e964973
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/src/adapters/methods/reft.py b/src/adapters/methods/reft.py
@@ -27,7 +27,12 @@ def __init__(
         projection = nn.Linear(in_dim, r_dim, bias=False, dtype=dtype)
         if orthogonal:
             # orthogonal is not implemented for half precision
-            projection = projection.to(dtype=torch.float32)
+            if dtype in [torch.float16, torch.bfloat16]:
+                warnings.warn(
+                    "Orthogonal parametrization is not supported for half precision dtypes. Converting REFT projection layer to float32.",
+                    UserWarning
+                )
+                projection = projection.to(dtype=torch.float32)
             self.projection = nn.utils.parametrizations.orthogonal(projection)
         else:
             self.projection = projection