[cherrypick][inference Zero-Dim] Support 0-Dim Tensor in Paddle-Tenso…

…rRT (#53752) * scale, square, sum, swish trt op converter support zero dim (#53660) * [Paddle-Inference] Support trt 0dims of expand_as_v2 and mish. (#53627) * support_expand_mish * add unitest for reshpe 0 dims (#53685) * Add trt pow converter. (#53462) * Add trt pow converter. * update to use AddConstantLayer * add dims=0 ut * [inference Zero-Dim]add equal, elementwise_op trt 0d (#53704) * [inference Zero-Dim]prelu trt converter support zero dim tensor (#53634) * prelu op trt converter support zero dim * [Inference Zero-Dim] Support trt 0dim of gelu, hard_swish, hard_sigmoid and leaky_relu (#53714) * support_act * delete_silu * [inference zero dim] softmax, stack op trt converter support zero dim (#53729) * softmax support * support stack * remove unused code * update --------- Co-authored-by: Yuanle Liu <yuanlehome@163.com> Co-authored-by: xiaoxiaohehe001 <49090790+xiaoxiaohehe001@users.noreply.github.com> Co-authored-by: zhoutianzi666 <39978853+zhoutianzi666@users.noreply.github.com> Co-authored-by: Wilber <jiweibo@baidu.com>
PaddlePaddle · May 13, 2023 · 20fbafe · 20fbafe
1 parent 286fd57
commit 20fbafe
Show file tree

Hide file tree

Showing 22 changed files with 879 additions and 433 deletions.
diff --git a/paddle/fluid/framework/ir/trt_support_nhwc_pass.cc b/paddle/fluid/framework/ir/trt_support_nhwc_pass.cc
@@ -356,6 +356,8 @@ void TrtSupportNHWCPass::ApplyImpl(Graph *graph) const {
     }
   };
   InsertTransposeOp();
+
+  AddStatis(transposed_ops.size());
 }
 
 }  // namespace ir

diff --git a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc
@@ -319,6 +319,37 @@ class ElementwiseTensorModOpConverter : public ElementwiseTensorOpConverter {
  public:
   ElementwiseTensorModOpConverter() { op_type_ = "mod"; }
 };
+
+// The diff between `pow` and `elementwise_pow` is in:
+// https://github.com/PaddlePaddle/Paddle/blob/release/2.4/python/paddle/tensor/math.py#L420
+class PowOpConverter : public OpConverter {
+ public:
+  PowOpConverter() {}
+  void operator()(const framework::proto::OpDesc& op,
+                  const framework::Scope& scope,
+                  bool test_mode) override {
+    VLOG(3) << "Convert a pow op to TensorRT IElementWiseLayer";
+    framework::OpDesc op_desc(op, nullptr);
+    auto* X = engine_->GetITensor(op_desc.Input("X").front());
+    float factor = PADDLE_GET_CONST(float, op_desc.GetAttr("factor"));
+    nvinfer1::Dims dims_x = X->getDimensions();
+    auto output_name = op_desc.Output("Out")[0];
+
+    nvinfer1::Dims trt_dims_y;
+    trt_dims_y.nbDims = dims_x.nbDims;
+    for (int i = 0; i < trt_dims_y.nbDims; i++) {
+      trt_dims_y.d[i] = 1;
+    }
+
+    std::vector<float> w_data{factor};
+    auto* Y = AddConstantLayer(w_data.data(), trt_dims_y);
+
+    auto* layer = TRT_ENGINE_ADD_LAYER(
+        engine_, ElementWise, *X, *Y, nvinfer1::ElementWiseOperation::kPOW);
+    RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode);
+  }
+};
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
@@ -369,3 +400,5 @@ REGISTER_TRT_OP_CONVERTER(logical_and, ElementwiseTensorLogicalAndOpConverter);
 REGISTER_TRT_OP_CONVERTER(less_equal, ElementwiseTensorLessEqualOpConverter);
 REGISTER_TRT_OP_CONVERTER(greater_equal,
                           ElementwiseTensorGreaterEqualOpConverter);
+
+REGISTER_TRT_OP_CONVERTER(pow, PowOpConverter);
diff --git a/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc
@@ -83,7 +83,10 @@ class ExpandOpConverter : public OpConverter {
       input_shape_tensor = Shape(input);
     }
 
-    auto* newInputTensor = Reshape(input, input_shape_tensor);
+    auto* newInputTensor =
+        Reshape(input,
+                input_shape_tensor,
+                ("expand_v2: reshape: (Output(" + output_name + ")").c_str());
 
     std::vector<int32_t> start_vec(shape_rank, 0);
     nvinfer1::Dims start;

diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h
@@ -407,13 +407,11 @@ class OpConverter {
   }
 
   nvinfer1::ITensor* Reshape(nvinfer1::ITensor* input,
-                             nvinfer1::ITensor* newShape) {
-    nvinfer1::ITensor* oldShape = Shape(input);
-    if (oldShape == newShape) {
-      return input;
-    }
+                             nvinfer1::ITensor* newShape,
+                             const std::string& name = "reshape") {
     auto* shuffle = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input);
     shuffle->setInput(1, *newShape);
+    shuffle->setName(name.c_str());
     return shuffle->getOutput(0);
   }
 

diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
@@ -87,7 +87,6 @@ class PReluOpConverter : public OpConverter {
           if (hw_tensor != nullptr) {
             shape_tensor = Concat(
                 std::vector<nvinfer1::ITensor*>{n_tensor, c_tensor, hw_tensor});
-
           } else {
             shape_tensor =
                 Concat(std::vector<nvinfer1::ITensor*>{n_tensor, c_tensor});

diff --git a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc
@@ -38,8 +38,23 @@ class SoftMaxOpConverter : public OpConverter {
                    ? PADDLE_GET_CONST(int, op_desc.GetAttr("axis"))
                    : -1;
 
-    auto* layer = TRT_ENGINE_ADD_LAYER(
-        engine_, SoftMax, *const_cast<nvinfer1::ITensor*>(input1));
+    // support 0 or 1 dims input
+    bool is_0_dims = input_dims == 0;
+    bool is_1_dims = input_dims == 1;
+    if (is_0_dims || is_1_dims) {
+      auto reshaped_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input1);
+      nvinfer1::Dims reshaped_dims;
+      reshaped_dims.nbDims = 2;
+      reshaped_dims.d[0] = 1;
+      reshaped_dims.d[1] = is_0_dims ? 1 : input_shape.d[0];
+      reshaped_layer->setReshapeDimensions(reshaped_dims);
+      input1 = reshaped_layer->getOutput(0);
+      input_shape = input1->getDimensions();
+      input_dims = input_shape.nbDims;
+      axis = -1;
+    }
+
+    auto* layer = TRT_ENGINE_ADD_LAYER(engine_, SoftMax, *input1);
     uint32_t axes = std::max(0, input_dims - 3);
     // TODO(cryoco): Poor workaround. Fix padded dims problem when TRT layers
     // support Nd.
@@ -68,11 +83,22 @@ class SoftMaxOpConverter : public OpConverter {
       }
     }
     layer->setAxes(1 << axes);
-    auto output_name = op_desc.Output("Out")[0];
-    RreplenishLayerAndOutput(layer, "softmax", {output_name}, test_mode);
 
     // The trt will not run int for softmax.
     engine_->SetTensorDynamicRange(input1, 1.0);
+    auto output_name = op_desc.Output("Out")[0];
+
+    // support 0 or 1 dims input
+    if (is_0_dims || is_1_dims) {
+      auto reshaped_layer =
+          TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *layer->getOutput(0));
+      reshaped_layer->setReshapeDimensions(
+          engine_->GetITensor(op_desc.Input("X")[0])->getDimensions());
+      RreplenishLayerAndOutput(
+          reshaped_layer, "reshape_softmax_reshape", {output_name}, test_mode);
+    } else {
+      RreplenishLayerAndOutput(layer, "softmax", {output_name}, test_mode);
+    }
   }
 };
 

diff --git a/paddle/fluid/inference/tensorrt/convert/stack_op.cc b/paddle/fluid/inference/tensorrt/convert/stack_op.cc
@@ -65,12 +65,11 @@ class StackOpConverter : public OpConverter {
     auto* after_shape_tensor = Concat(shape_tensor_vec);
 
     for (int i = 0; i < input_num; ++i) {
-      auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *inputs[i]);
-      reshape_layer->setInput(1, *after_shape_tensor);
-      inputs[i] = reshape_layer->getOutput(0);
-      reshape_layer->setName(("stack: reshape: (Output( " + std::to_string(i) +
-                              " )" + output_name + ")")
-                                 .c_str());
+      inputs[i] = Reshape(inputs[i],
+                          after_shape_tensor,
+                          ("stack: reshape: (Output( " + std::to_string(i) +
+                           " )" + output_name + ")")
+                              .c_str());
     }
 
     auto* layer = TRT_ENGINE_ADD_LAYER(