diff --git a/xformers/csrc/swiglu/swiglu_packedw.cpp b/xformers/csrc/swiglu/swiglu_packedw.cpp
index d27241730f..a30bcb200b 100644
--- a/xformers/csrc/swiglu/swiglu_packedw.cpp
+++ b/xformers/csrc/swiglu/swiglu_packedw.cpp
@@ -111,11 +111,11 @@ class SwiGLUPackedWeights
   static torch::autograd::variable_list backward(
       torch::autograd::AutogradContext* ctx,
       torch::autograd::variable_list grad_outputs) {
+    auto saved = ctx->get_saved_variables();
     at::AutoDispatchBelowADInplaceOrView g;
 
     // Unpack variables
     auto dx5 = grad_outputs[0];
-    auto saved = ctx->get_saved_variables();
     auto x = saved[0];
     auto w1w2 = saved[1];
     auto w3 = saved[2];