diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index f66a48ab771a8..b510b2ab5f994 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -949,9 +949,11 @@ void AnalysisConfig::Update() {
   //  Case3: pass_builder_ has been created and belongs to
   // GpuPassStrategy(or IpuPassStrategy), neither enable mkldnn and
   // disable mkldnn will be executed
-  if (!use_gpu() && !use_xpu() && !use_ipu() && !use_custom_device() &&
-      !use_mkldnn_) {
-    // User manually disable mkldnn
+  if ((!use_gpu() && !use_xpu() && !use_ipu() && !use_mkldnn_) ||
+      (use_mkldnn_ &&
+       !phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2))) {
+    // User manually disable mkldnn or disable when not support AVX2
+    use_mkldnn_ = false;
     pass_builder()->DisableMKLDNN();
   }
 #endif
diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h
index 3dad3a68c1513..f7c0c7dff93a9 100644
--- a/paddle/fluid/inference/api/paddle_analysis_config.h
+++ b/paddle/fluid/inference/api/paddle_analysis_config.h
@@ -39,7 +39,6 @@
 #include "paddle_api.h"           // NOLINT
 #include "paddle_pass_builder.h"  // NOLINT
 #ifdef PADDLE_WITH_DNNL
-#include "paddle/phi/backends/cpu/cpu_info.h"
 #include "paddle_mkldnn_quantizer_config.h"  // NOLINT
 #endif
 
@@ -1302,13 +1301,10 @@ struct PD_INFER_DECL AnalysisConfig {
   std::unordered_set<std::string> trt_ops_run_float_;
 
 #ifdef PADDLE_WITH_DNNL
-  bool use_mkldnn_{
-      phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2) ? true
-                                                                       : false};
+  bool use_mkldnn_{true};
 #else
   bool use_mkldnn_{false};
 #endif
-
   std::unordered_set<std::string> mkldnn_enabled_op_types_;
 
   bool model_from_memory_{false};