From aa869ac221b909dc47c75d1a591e728e43bed4be Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Wed, 22 Nov 2023 18:57:37 +0800 Subject: [PATCH 1/9] Fix cpu.h header can not package in deploy --- .../inference/api/paddle_analysis_config.h | 52 +++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 3dad3a68c1513..8ca13d72bae45 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -39,10 +39,11 @@ #include "paddle_api.h" // NOLINT #include "paddle_pass_builder.h" // NOLINT #ifdef PADDLE_WITH_DNNL -#include "paddle/phi/backends/cpu/cpu_info.h" #include "paddle_mkldnn_quantizer_config.h" // NOLINT #endif - +#ifdef PADDLE_WITH_XBYAK +#include "xbyak/xbyak_util.h" +#endif namespace paddle { class AnalysisPredictor; @@ -1302,9 +1303,50 @@ struct PD_INFER_DECL AnalysisConfig { std::unordered_set trt_ops_run_float_; #ifdef PADDLE_WITH_DNNL - bool use_mkldnn_{ - phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2) ? true - : false}; +#ifdef PADDLE_WITH_XBYAK + bool SupportAVX2() { + using namespace Xbyak::util; // NOLINT + Xbyak::util::Cpu cpu; + return cpu.has(Cpu::tAVX2); + } +#else + bool SupportAVX2() { +#ifdef _WIN32 +#define cpuid(reg, x) __cpuidex(reg, x, 0) +#else +#if !defined(WITH_NV_JETSON) && !defined(PADDLE_WITH_ARM) && \ + !defined(PADDLE_WITH_SW) && !defined(PADDLE_WITH_MIPS) && \ + !defined(PADDLE_WITH_LOONGARCH) +#include + inline void cpuid(int reg[4], int x) { + __cpuid_count(x, 0, reg[0], reg[1], reg[2], reg[3]); + } +#endif +#endif + +#if !defined(WITH_NV_JETSON) && !defined(PADDLE_WITH_ARM) && \ + !defined(PADDLE_WITH_SW) && !defined(PADDLE_WITH_MIPS) && \ + !defined(PADDLE_WITH_LOONGARCH) + std::array reg; + cpuid(reg.data(), 0); + int nIds = reg[0]; + if (nIds >= 0x00000001) { + // EAX = 1 + cpuid(reg.data(), 0x00000001); + // AVX: ECX Bit 28 + } + if (nIds >= 0x00000007) { + // EAX = 7 + cpuid(reg.data(), 0x00000007); + // AVX2: EBX Bit 5 + int avx2_mask = (1 << 5); + return (reg[1] & avx2_mask) != 0; + } +#endif + return false; + } +#endif + bool use_mkldnn_{SupportAVX2() ? true : false}; #else bool use_mkldnn_{false}; #endif From d52814fc0eade91601574924915491510bc689da Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Wed, 22 Nov 2023 19:48:07 +0800 Subject: [PATCH 2/9] run CI --- paddle/fluid/inference/api/paddle_analysis_config.h | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 8ca13d72bae45..6edc80666b474 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -1333,7 +1333,6 @@ struct PD_INFER_DECL AnalysisConfig { if (nIds >= 0x00000001) { // EAX = 1 cpuid(reg.data(), 0x00000001); - // AVX: ECX Bit 28 } if (nIds >= 0x00000007) { // EAX = 7 From f56e9f509646542ff28909481f2137d222956788 Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 06:29:39 +0800 Subject: [PATCH 3/9] Add win32 header --- .../inference/api/paddle_analysis_config.h | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 6edc80666b474..e2dd8f5bd5440 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -44,6 +44,27 @@ #ifdef PADDLE_WITH_XBYAK #include "xbyak/xbyak_util.h" #endif + +#ifdef _WIN32 +#if defined(__AVX2__) +#include // avx2 +#elif defined(__AVX__) +#include // avx +#endif // AVX +#else // WIN32 +#ifdef __AVX__ +#include +#endif +#endif // WIN32 + +#if defined(_WIN32) +#define ALIGN32_BEG __declspec(align(32)) +#define ALIGN32_END +#else +#define ALIGN32_BEG +#define ALIGN32_END __attribute__((aligned(32))) +#endif // _WIN32 + namespace paddle { class AnalysisPredictor; From 375cbd941ffcacb8efa7618cbf7a8838e858a5ae Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 11:45:06 +0800 Subject: [PATCH 4/9] enable mkldnn in predictor not in config --- .../fluid/inference/api/analysis_predictor.cc | 5 ++++ .../inference/api/paddle_analysis_config.h | 29 +------------------ 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index acad824fcc2f2..045562682eaf6 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -58,6 +58,7 @@ #include "paddle/fluid/platform/profiler.h" #include "paddle/phi/api/include/context_pool.h" #include "paddle/phi/api/include/tensor.h" +#include "paddle/phi/backends/cpu/cpu_info.h" #include "paddle/phi/common/backend.h" #include "paddle/phi/common/data_type.h" #include "paddle/phi/common/place.h" @@ -359,6 +360,10 @@ AnalysisPredictor::AnalysisPredictor(const AnalysisConfig &config) config_.SwitchIrOptim(false); } } + if (config_.ir_optim() && + phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2)) { + config_.EnableMKLDNN(); + } int trt_identifier = config_.trt_engine_memory_sharing_identifier_; if (trt_identifier > 0) { // NOTE(liuyuanle): For convenience, we set the id of the predictor to diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index e2dd8f5bd5440..9007bda366d28 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -41,29 +41,6 @@ #ifdef PADDLE_WITH_DNNL #include "paddle_mkldnn_quantizer_config.h" // NOLINT #endif -#ifdef PADDLE_WITH_XBYAK -#include "xbyak/xbyak_util.h" -#endif - -#ifdef _WIN32 -#if defined(__AVX2__) -#include // avx2 -#elif defined(__AVX__) -#include // avx -#endif // AVX -#else // WIN32 -#ifdef __AVX__ -#include -#endif -#endif // WIN32 - -#if defined(_WIN32) -#define ALIGN32_BEG __declspec(align(32)) -#define ALIGN32_END -#else -#define ALIGN32_BEG -#define ALIGN32_END __attribute__((aligned(32))) -#endif // _WIN32 namespace paddle { @@ -1365,12 +1342,8 @@ struct PD_INFER_DECL AnalysisConfig { #endif return false; } -#endif - bool use_mkldnn_{SupportAVX2() ? true : false}; -#else - bool use_mkldnn_{false}; -#endif + bool use_mkldnn_{false}; std::unordered_set mkldnn_enabled_op_types_; bool model_from_memory_{false}; From ad536fbc14bf213e82e92f261107edaf554595f9 Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 12:23:35 +0800 Subject: [PATCH 5/9] delete useless code --- .../inference/api/paddle_analysis_config.h | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 9007bda366d28..77d16a6e414cb 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -1300,49 +1300,6 @@ struct PD_INFER_DECL AnalysisConfig { std::unordered_set trt_ops_run_float_; -#ifdef PADDLE_WITH_DNNL -#ifdef PADDLE_WITH_XBYAK - bool SupportAVX2() { - using namespace Xbyak::util; // NOLINT - Xbyak::util::Cpu cpu; - return cpu.has(Cpu::tAVX2); - } -#else - bool SupportAVX2() { -#ifdef _WIN32 -#define cpuid(reg, x) __cpuidex(reg, x, 0) -#else -#if !defined(WITH_NV_JETSON) && !defined(PADDLE_WITH_ARM) && \ - !defined(PADDLE_WITH_SW) && !defined(PADDLE_WITH_MIPS) && \ - !defined(PADDLE_WITH_LOONGARCH) -#include - inline void cpuid(int reg[4], int x) { - __cpuid_count(x, 0, reg[0], reg[1], reg[2], reg[3]); - } -#endif -#endif - -#if !defined(WITH_NV_JETSON) && !defined(PADDLE_WITH_ARM) && \ - !defined(PADDLE_WITH_SW) && !defined(PADDLE_WITH_MIPS) && \ - !defined(PADDLE_WITH_LOONGARCH) - std::array reg; - cpuid(reg.data(), 0); - int nIds = reg[0]; - if (nIds >= 0x00000001) { - // EAX = 1 - cpuid(reg.data(), 0x00000001); - } - if (nIds >= 0x00000007) { - // EAX = 7 - cpuid(reg.data(), 0x00000007); - // AVX2: EBX Bit 5 - int avx2_mask = (1 << 5); - return (reg[1] & avx2_mask) != 0; - } -#endif - return false; - } - bool use_mkldnn_{false}; std::unordered_set mkldnn_enabled_op_types_; From 376e287e1f86748caac9f50175b157d96bba9a7d Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 14:18:03 +0800 Subject: [PATCH 6/9] change logic to filter avx --- paddle/fluid/inference/api/analysis_config.cc | 7 ++++--- paddle/fluid/inference/api/analysis_predictor.cc | 5 ----- paddle/fluid/inference/api/paddle_analysis_config.h | 4 ++++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index f66a48ab771a8..381470cc16abd 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -949,9 +949,10 @@ void AnalysisConfig::Update() { // Case3: pass_builder_ has been created and belongs to // GpuPassStrategy(or IpuPassStrategy), neither enable mkldnn and // disable mkldnn will be executed - if (!use_gpu() && !use_xpu() && !use_ipu() && !use_custom_device() && - !use_mkldnn_) { - // User manually disable mkldnn + if ((!use_gpu() && !use_xpu() && !use_ipu() && !use_mkldnn_) || + (!phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2) && + !use_mkldnn_)) { + // User manually disable mkldnn or disable when not support AVX2 pass_builder()->DisableMKLDNN(); } #endif diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 045562682eaf6..acad824fcc2f2 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -58,7 +58,6 @@ #include "paddle/fluid/platform/profiler.h" #include "paddle/phi/api/include/context_pool.h" #include "paddle/phi/api/include/tensor.h" -#include "paddle/phi/backends/cpu/cpu_info.h" #include "paddle/phi/common/backend.h" #include "paddle/phi/common/data_type.h" #include "paddle/phi/common/place.h" @@ -360,10 +359,6 @@ AnalysisPredictor::AnalysisPredictor(const AnalysisConfig &config) config_.SwitchIrOptim(false); } } - if (config_.ir_optim() && - phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2)) { - config_.EnableMKLDNN(); - } int trt_identifier = config_.trt_engine_memory_sharing_identifier_; if (trt_identifier > 0) { // NOTE(liuyuanle): For convenience, we set the id of the predictor to diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h index 77d16a6e414cb..f7c0c7dff93a9 100644 --- a/paddle/fluid/inference/api/paddle_analysis_config.h +++ b/paddle/fluid/inference/api/paddle_analysis_config.h @@ -1300,7 +1300,11 @@ struct PD_INFER_DECL AnalysisConfig { std::unordered_set trt_ops_run_float_; +#ifdef PADDLE_WITH_DNNL + bool use_mkldnn_{true}; +#else bool use_mkldnn_{false}; +#endif std::unordered_set mkldnn_enabled_op_types_; bool model_from_memory_{false}; From d97324f01cde8adaac250df9e5c59f0524eb87d9 Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 14:23:56 +0800 Subject: [PATCH 7/9] fix --- paddle/fluid/inference/api/analysis_config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 381470cc16abd..1affaafe944ad 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -951,7 +951,7 @@ void AnalysisConfig::Update() { // disable mkldnn will be executed if ((!use_gpu() && !use_xpu() && !use_ipu() && !use_mkldnn_) || (!phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2) && - !use_mkldnn_)) { + use_mkldnn_)) { // User manually disable mkldnn or disable when not support AVX2 pass_builder()->DisableMKLDNN(); } From 6975949ad35c66354389fe28f54e629a45f3797d Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 14:32:31 +0800 Subject: [PATCH 8/9] make use_mkldnn_ = false --- paddle/fluid/inference/api/analysis_config.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 1affaafe944ad..ef82488ef7485 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -953,6 +953,7 @@ void AnalysisConfig::Update() { (!phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2) && use_mkldnn_)) { // User manually disable mkldnn or disable when not support AVX2 + use_mkldnn_ = false; pass_builder()->DisableMKLDNN(); } #endif From ee27949097b0567a6a876165a987fb419af647c0 Mon Sep 17 00:00:00 2001 From: zhanglirong1999 <1695074375@qq.com> Date: Thu, 23 Nov 2023 16:51:29 +0800 Subject: [PATCH 9/9] exchange position to coverage CI pass --- paddle/fluid/inference/api/analysis_config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index ef82488ef7485..b510b2ab5f994 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -950,8 +950,8 @@ void AnalysisConfig::Update() { // GpuPassStrategy(or IpuPassStrategy), neither enable mkldnn and // disable mkldnn will be executed if ((!use_gpu() && !use_xpu() && !use_ipu() && !use_mkldnn_) || - (!phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2) && - use_mkldnn_)) { + (use_mkldnn_ && + !phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx2))) { // User manually disable mkldnn or disable when not support AVX2 use_mkldnn_ = false; pass_builder()->DisableMKLDNN();