From 29a9fb3c584b6f243b501a124d35e029aee15dd4 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Fri, 5 Aug 2022 03:35:08 +0000 Subject: [PATCH 01/15] commit --- .../fluid/inference/api/analysis_predictor.cc | 1 + .../inference/tensorrt/convert/CMakeLists.txt | 1 + .../tensorrt/convert/matmul_v2_op.cc | 125 ++++++++++++++++++ paddle/fluid/inference/tensorrt/op_teller.cc | 10 ++ 4 files changed, 137 insertions(+) create mode 100644 paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index cc169c930768dc..7cfce0627814e7 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -2035,6 +2035,7 @@ USE_TRT_CONVERTER(transpose2); USE_TRT_CONVERTER(flatten); USE_TRT_CONVERTER(flatten_contiguous_range); USE_TRT_CONVERTER(matmul); +USE_TRT_CONVERTER(matmul_v2); USE_TRT_CONVERTER(conv2d); USE_TRT_CONVERTER(relu); USE_TRT_CONVERTER(exp); diff --git a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt index 4a13b6c00ac004..bbe5cb78ae7a9a 100644 --- a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt @@ -3,6 +3,7 @@ list( APPEND CONVERT_FILES matmul_op.cc + matmul_v2_op.cc conv2d_op.cc fc_op.cc pool2d_op.cc diff --git a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc new file mode 100644 index 00000000000000..3b4ca35f49a060 --- /dev/null +++ b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc @@ -0,0 +1,125 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/tensorrt/convert/op_converter.h" +#include "paddle/fluid/inference/tensorrt/plugin/matmul_op_int8_plugin.h" + +namespace paddle { +namespace framework { +class Scope; + +namespace proto { +class OpDesc; +} // namespace proto +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace inference { +namespace tensorrt { + +/* + * MatMulOp, IMatrixMultiplyLayer in TRT. This Layer doesn't has weights. + */ +class MatMulV2OpConverter : public OpConverter { + public: + void operator()(const framework::proto::OpDesc& op, + const framework::Scope& scope, + bool test_mode) override { + VLOG(3) << "convert a fluid matmul op to tensorrt matmul layer "; + framework::OpDesc op_desc(op, nullptr); + nvinfer1::ILayer* layer = nullptr; + + auto* block = op_desc.Block(); + if (block) { + std::cout << "The block desc is nullptr " << std::endl; + } + std::cout << "The block desc is nullptr " << std::endl; + + // Declare inputs + auto* input1 = engine_->GetITensor(op_desc.Input("X")[0]); + auto* input2 = engine_->GetITensor(op_desc.Input("Y")[0]); + + nvinfer1::Dims dims_x = input1->getDimensions(); + nvinfer1::Dims dims_y = input2->getDimensions(); + + bool transpose_X = PADDLE_GET_CONST(bool, op_desc.GetAttr("trans_x")); + bool transpose_Y = PADDLE_GET_CONST(bool, op_desc.GetAttr("trans_y")); + + auto output_name = op_desc.Output("Out")[0]; + + nvinfer1::MatrixOperation matrix_operation_X = + transpose_X ? nvinfer1::MatrixOperation::kTRANSPOSE + : nvinfer1::MatrixOperation::kNONE; + nvinfer1::MatrixOperation matrix_operation_Y = + transpose_Y ? nvinfer1::MatrixOperation::kTRANSPOSE + : nvinfer1::MatrixOperation::kNONE; + + int one_num = 0; + nvinfer1::ITensor* new_shape_tensor = nullptr; + if (dims_x.nbDims < dims_y.nbDims) { + one_num = dims_y.nbDims - dims_x.nbDims; + new_shape_tensor = Shape(input1); + std::vector one_vec(one_num, 1); + auto* one_tensor = Add1DConstantLayer(one_vec); + new_shape_tensor = + Concat(std::vector{one_tensor, new_shape_tensor}); + + auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input1); + reshape_layer->setInput(1, *new_shape_tensor); + + layer = TRT_ENGINE_ADD_LAYER(engine_, + MatrixMultiply, + *reshape_layer->getOutput(0), + matrix_operation_X, + *input2, + matrix_operation_Y); + + } else if (dims_x.nbDims > dims_y.nbDims) { + one_num = dims_x.nbDims - dims_y.nbDims; + new_shape_tensor = Shape(input2); + std::vector one_vec(one_num, 1); + auto* one_tensor = Add1DConstantLayer(one_vec); + new_shape_tensor = + Concat(std::vector{one_tensor, new_shape_tensor}); + auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input2); + reshape_layer->setInput(1, *new_shape_tensor); + + layer = TRT_ENGINE_ADD_LAYER(engine_, + MatrixMultiply, + *input1, + matrix_operation_X, + *reshape_layer->getOutput(0), + matrix_operation_Y); + + } else { + layer = TRT_ENGINE_ADD_LAYER(engine_, + MatrixMultiply, + *input1, + matrix_operation_X, + *input2, + matrix_operation_Y); + } + VLOG(3) << "Convert a fluid matmul_op_float to TensorRT "; + + RreplenishLayerAndOutput( + layer, "matmul_op_float_no_alpha", {output_name}, test_mode); + } +}; + +} // namespace tensorrt +} // namespace inference +} // namespace paddle + +REGISTER_TRT_OP_CONVERTER(matmul_v2, MatMulV2OpConverter); diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index c602f2ff071240..57c10fdb0ea010 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -70,6 +70,7 @@ struct SimpleOpTypeSetTeller : public Teller { std::unordered_set int8_teller_set{ "mul", "matmul", + "matmul_v2", "conv2d", "conv2d_fusion", "pool2d", @@ -177,6 +178,7 @@ struct SimpleOpTypeSetTeller : public Teller { std::unordered_set teller_set{ "mul", "matmul", + "matmul_v2", "conv2d", "conv2d_fusion", "pool2d", @@ -525,6 +527,14 @@ bool OpTeller::Tell(const framework::ir::Node* node, } } + if (op_type == "matmul_v2") { + auto* block = desc.Block(); + if (block) { + std::cout << "The block desc is nullptr" << std::endl; + } + return true; + } + if (op_type == "matmul") { auto* block = desc.Block(); if (block == nullptr) { From a51b3bf70b4cb862b35ee31391e6f3a7c9d70a66 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Fri, 5 Aug 2022 05:27:22 +0000 Subject: [PATCH 02/15] commit --- .../tensorrt/convert/matmul_v2_op.cc | 6 - paddle/fluid/inference/tensorrt/op_teller.cc | 2 +- .../inference/test_trt_convert_matmul_v2.py | 124 ++++++++++++++++++ 3 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py diff --git a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc index 3b4ca35f49a060..0191c92001475a 100644 --- a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc @@ -41,12 +41,6 @@ class MatMulV2OpConverter : public OpConverter { framework::OpDesc op_desc(op, nullptr); nvinfer1::ILayer* layer = nullptr; - auto* block = op_desc.Block(); - if (block) { - std::cout << "The block desc is nullptr " << std::endl; - } - std::cout << "The block desc is nullptr " << std::endl; - // Declare inputs auto* input1 = engine_->GetITensor(op_desc.Input("X")[0]); auto* input2 = engine_->GetITensor(op_desc.Input("Y")[0]); diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 39837db54c995d..b0c4b229007981 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -534,7 +534,7 @@ bool OpTeller::Tell(const framework::ir::Node* node, if (op_type == "matmul_v2") { auto* block = desc.Block(); if (block) { - std::cout << "The block desc is nullptr" << std::endl; + //std::cout << "The block desc is nullptr" << std::endl; } return true; } diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py new file mode 100644 index 00000000000000..04ef83ecfc5da0 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -0,0 +1,124 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from trt_layer_auto_scan_test import TrtLayerAutoScanTest, SkipReasons +from program_config import TensorConfig, ProgramConfig +import numpy as np +import paddle.inference as paddle_infer +from functools import partial +from typing import Optional, List, Callable, Dict, Any, Set +import unittest + +class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): + + def is_program_valid(self, program_config: ProgramConfig) -> bool: + return True + + def sample_program_configs(self): + + def generate_input(shape): + return np.random.random(shape).astype(np.float32) + + for trans_x in [True]: + for trans_y in [True]: + if trans_x and trans_y: + input1_shape = [4, 4, 4] + input2_shape = [4, 4] + # if trans_x and not trans_y: + # input1_shape = [4, 4, 4] + # input2_shape = [4, 4, 4] + # if not trans_x and trans_y: + # input1_shape = [batch, 32, 6] + # input2_shape = [batch, 11, 6] + # if not trans_x and not trans_y: + # input1_shape = [batch, 32, 6] + # input2_shape = [batch, 6, 11] + for alpha in [0.3, 1.0]: + dics = [{ + "transpose_X": trans_x, + "transpose_Y": trans_y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [] + }] + ops_config = [{ + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"] + }, + "op_outputs": { + "Out": ["output_data"] + }, + "op_attrs": dics[0] + }] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input1_data": + TensorConfig( + data_gen=partial(generate_input, input1_shape)), + "input2_data": + TensorConfig( + data_gen=partial(generate_input, input2_shape)) + }, + outputs=["output_data"]) + + yield program_config + + def sample_predictor_configs( + self, program_config) -> (paddle_infer.Config, List[int], float): + + def generate_dynamic_shape(attrs): + self.dynamic_shape.min_input_shape = { + "input1_data": [1, 4, 4], + "input2_data": [4, 4] + } + self.dynamic_shape.max_input_shape = { + "input1_data": [16, 4, 4], + "input2_data": [4, 4] + } + self.dynamic_shape.opt_input_shape = { + "input1_data": [8, 4, 4], + "input2_data": [4, 4] + } + + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + + # for dynamic_shape + generate_dynamic_shape(attrs) + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + yield self.create_inference_config(), (1, 3), 1e-5 + self.trt_param.precision = paddle_infer.PrecisionType.Half + yield self.create_inference_config(), (1, 3), 1e-5 + + def add_skip_trt_case(self): + pass + + def test(self): + self.add_skip_trt_case() + self.run_test() + + +if __name__ == "__main__": + unittest.main() From 501a1c52294a8f1977860a094810483d027bc4d7 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Fri, 5 Aug 2022 05:57:37 +0000 Subject: [PATCH 03/15] commit --- paddle/fluid/inference/tensorrt/op_teller.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index b0c4b229007981..653efa33f31418 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -533,8 +533,9 @@ bool OpTeller::Tell(const framework::ir::Node* node, if (op_type == "matmul_v2") { auto* block = desc.Block(); - if (block) { + if (block == nullptr) { //std::cout << "The block desc is nullptr" << std::endl; + return false; } return true; } From 02fd965b6f63ae0807954af7d3002a948d6f89c3 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Fri, 2 Sep 2022 03:38:40 +0000 Subject: [PATCH 04/15] clean code --- .../inference/test_trt_convert_matmul_v2.py | 77 +++++++++---------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 04ef83ecfc5da0..3d45f9ddf2679e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -20,6 +20,7 @@ from typing import Optional, List, Callable, Dict, Any, Set import unittest + class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): def is_program_valid(self, program_config: ProgramConfig) -> bool: @@ -44,45 +45,43 @@ def generate_input(shape): # if not trans_x and not trans_y: # input1_shape = [batch, 32, 6] # input2_shape = [batch, 6, 11] - for alpha in [0.3, 1.0]: - dics = [{ - "transpose_X": trans_x, - "transpose_Y": trans_y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] - ops = self.generate_op_config(ops_config) - - program_config = ProgramConfig( - ops=ops, - weights={}, - inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) - }, - outputs=["output_data"]) - - yield program_config + dics = [{ + "transpose_X": trans_x, + "transpose_Y": trans_y, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [] + }] + ops_config = [{ + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"] + }, + "op_outputs": { + "Out": ["output_data"] + }, + "op_attrs": dics[0] + }] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input1_data": + TensorConfig( + data_gen=partial(generate_input, input1_shape)), + "input2_data": + TensorConfig( + data_gen=partial(generate_input, input2_shape)) + }, + outputs=["output_data"]) + + yield program_config def sample_predictor_configs( self, program_config) -> (paddle_infer.Config, List[int], float): From 4b9117a80b60918e5393a04b1f9d7db9fda388d8 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Wed, 14 Sep 2022 07:36:26 +0000 Subject: [PATCH 05/15] commit --- paddle/fluid/inference/tensorrt/op_teller.cc | 4 ++- .../inference/test_trt_convert_matmul_v2.py | 28 ++++--------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index de93fe487876df..f6ac0ca5168805 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -535,7 +535,9 @@ bool OpTeller::Tell(const framework::ir::Node* node, if (op_type == "matmul_v2") { auto* block = desc.Block(); if (block == nullptr) { - //std::cout << "The block desc is nullptr" << std::endl; + VLOG(3) << "The block desc is nullptr, we can't continue to analyze. " + "Developers need to check whether block_desc is passed in " + "the pass."; return false; } return true; diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 3d45f9ddf2679e..1e92175505aea4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -31,29 +31,13 @@ def sample_program_configs(self): def generate_input(shape): return np.random.random(shape).astype(np.float32) - for trans_x in [True]: - for trans_y in [True]: - if trans_x and trans_y: - input1_shape = [4, 4, 4] - input2_shape = [4, 4] - # if trans_x and not trans_y: - # input1_shape = [4, 4, 4] - # input2_shape = [4, 4, 4] - # if not trans_x and trans_y: - # input1_shape = [batch, 32, 6] - # input2_shape = [batch, 11, 6] - # if not trans_x and not trans_y: - # input1_shape = [batch, 32, 6] - # input2_shape = [batch, 6, 11] + for trans_x in [True, False]: + for trans_y in [True, False]: + input1_shape = [4, 4, 4] + input2_shape = [4, 4] dics = [{ - "transpose_X": trans_x, - "transpose_Y": trans_y, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] + "trans_x": trans_x, + "trans_y": trans_y, }] ops_config = [{ "op_type": "matmul_v2", From 7bb74b404944e1cdefce08226ea88cc5fb9068b6 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Wed, 14 Sep 2022 07:40:18 +0000 Subject: [PATCH 06/15] add commit --- .../tensorrt/convert/matmul_v2_op.cc | 2 +- paddle/fluid/inference/tensorrt/op_teller.cc | 3 + .../inference/test_trt_convert_matmul_v2.py | 82 +++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc index 0191c92001475a..683c23e9206c3d 100644 --- a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc @@ -37,7 +37,7 @@ class MatMulV2OpConverter : public OpConverter { void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { - VLOG(3) << "convert a fluid matmul op to tensorrt matmul layer "; + VLOG(3) << "convert a fluid matmul_v2 op to tensorrt matmul layer "; framework::OpDesc op_desc(op, nullptr); nvinfer1::ILayer* layer = nullptr; diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index f6ac0ca5168805..ed9a8d7a20de86 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -533,6 +533,9 @@ bool OpTeller::Tell(const framework::ir::Node* node, } if (op_type == "matmul_v2") { + if (!with_dynamic_shape) { + return false; + } auto* block = desc.Block(); if (block == nullptr) { VLOG(3) << "The block desc is nullptr, we can't continue to analyze. " diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 1e92175505aea4..427bf11ff98bc4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -103,5 +103,87 @@ def test(self): self.run_test() +class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): + + def is_program_valid(self, program_config: ProgramConfig) -> bool: + return True + + def sample_program_configs(self): + + def generate_input(shape): + return np.random.random(shape).astype(np.float32) + + for trans_x in [False]: + for trans_y in [False]: + input1_shape = [6, 4] + input2_shape = [4, 4, 9] + dics = [{ + "trans_x": trans_x, + "trans_y": trans_y, + }] + ops_config = [{ + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"] + }, + "op_outputs": { + "Out": ["output_data"] + }, + "op_attrs": dics[0] + }] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input1_data": + TensorConfig( + data_gen=partial(generate_input, input1_shape)), + "input2_data": + TensorConfig( + data_gen=partial(generate_input, input2_shape)) + }, + outputs=["output_data"]) + + yield program_config + + def sample_predictor_configs( + self, program_config) -> (paddle_infer.Config, List[int], float): + + def generate_dynamic_shape(attrs): + self.dynamic_shape.min_input_shape = { + "input1_data": [6, 4], + "input2_data": [4, 4, 9] + } + self.dynamic_shape.max_input_shape = { + "input1_data": [6, 4], + "input2_data": [4, 4, 9] + } + self.dynamic_shape.opt_input_shape = { + "input1_data": [6, 4], + "input2_data": [4, 4, 9] + } + + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + + # for dynamic_shape + generate_dynamic_shape(attrs) + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + yield self.create_inference_config(), (1, 3), 1e-5 + self.trt_param.precision = paddle_infer.PrecisionType.Half + yield self.create_inference_config(), (1, 3), 1e-5 + + def add_skip_trt_case(self): + pass + + def test(self): + self.add_skip_trt_case() + self.run_test() + + if __name__ == "__main__": unittest.main() From 3bf4fb4609a22b66591ecf8919f582404c633ff9 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Thu, 15 Sep 2022 06:18:28 +0000 Subject: [PATCH 07/15] commit --- .../inference/test_trt_convert_matmul_v2.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 427bf11ff98bc4..c4eea2bcef42a7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -19,6 +19,7 @@ from functools import partial from typing import Optional, List, Callable, Dict, Any, Set import unittest +import os class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): @@ -88,12 +89,18 @@ def generate_dynamic_shape(attrs): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] + # The output has little diff between gpu and trt in CI-Windows-Inference + tol_fp32 = 1e-5 + tol_half = 1e-5 + if (os.name == 'nt'): + tol_fp32 = 1e-3 + tol_half = 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 - yield self.create_inference_config(), (1, 3), 1e-5 + yield self.create_inference_config(), (1, 3), tol_fp32 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), (1, 3), 1e-5 + yield self.create_inference_config(), (1, 3), tol_half def add_skip_trt_case(self): pass @@ -169,13 +176,18 @@ def generate_dynamic_shape(attrs): attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - + # The output has little diff between gpu and trt in CI-Windows-Inference + tol_fp32 = 1e-5 + tol_half = 1e-5 + if (os.name == 'nt'): + tol_fp32 = 1e-3 + tol_half = 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 - yield self.create_inference_config(), (1, 3), 1e-5 + yield self.create_inference_config(), (1, 3), tol_fp32 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), (1, 3), 1e-5 + yield self.create_inference_config(), (1, 3), tol_half def add_skip_trt_case(self): pass From 548048639ce54b78f7a2ca9eeaefbdf19d9a544e Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Thu, 15 Sep 2022 06:28:12 +0000 Subject: [PATCH 08/15] commit --- .../inference/test_trt_convert_matmul_v2.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index c4eea2bcef42a7..d957b08f170d13 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -32,10 +32,10 @@ def sample_program_configs(self): def generate_input(shape): return np.random.random(shape).astype(np.float32) - for trans_x in [True, False]: - for trans_y in [True, False]: - input1_shape = [4, 4, 4] - input2_shape = [4, 4] + for trans_x in [False]: + for trans_y in [False]: + input1_shape = [15, 64, 350, 75] + input2_shape = [75, 25] dics = [{ "trans_x": trans_x, "trans_y": trans_y, @@ -73,16 +73,16 @@ def sample_predictor_configs( def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { - "input1_data": [1, 4, 4], - "input2_data": [4, 4] + "input1_data": [10, 64, 350, 75], + "input2_data": [75, 25] } self.dynamic_shape.max_input_shape = { - "input1_data": [16, 4, 4], - "input2_data": [4, 4] + "input1_data": [100, 64, 350, 75], + "input2_data": [75, 25] } self.dynamic_shape.opt_input_shape = { - "input1_data": [8, 4, 4], - "input2_data": [4, 4] + "input1_data": [15, 64, 350, 75], + "input2_data": [75, 25] } attrs = [ From 490d15dd62942e927feee8a2d23fffb8833a99ad Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Fri, 16 Sep 2022 07:35:56 +0000 Subject: [PATCH 09/15] commit --- .../tests/unittests/ir/inference/test_trt_convert_matmul_v2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index d957b08f170d13..05aba9a3e09363 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -112,9 +112,6 @@ def test(self): class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: - return True - def sample_program_configs(self): def generate_input(shape): From 4fba2b12d769b2874cb1074cb6c648c90e928b66 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Sun, 18 Sep 2022 11:18:23 +0000 Subject: [PATCH 10/15] commit --- .../tests/unittests/ir/inference/test_trt_convert_matmul_v2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 05aba9a3e09363..73d06d7d3ef89b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -24,9 +24,6 @@ class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: - return True - def sample_program_configs(self): def generate_input(shape): From 431f56b0ee709badcd9039dee9c1b7002eeb1dc6 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Mon, 19 Sep 2022 03:23:23 +0000 Subject: [PATCH 11/15] commit --- .../inference/test_trt_convert_matmul_v2.py | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 73d06d7d3ef89b..9cf769a99161b4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -28,42 +28,42 @@ def sample_program_configs(self): def generate_input(shape): return np.random.random(shape).astype(np.float32) - - for trans_x in [False]: - for trans_y in [False]: - input1_shape = [15, 64, 350, 75] - input2_shape = [75, 25] - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] - ops = self.generate_op_config(ops_config) - - program_config = ProgramConfig( - ops=ops, - weights={}, - inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) - }, - outputs=["output_data"]) - - yield program_config + for batch in [10, 11, 12, 13, 14, 15]: + for trans_x in [False]: + for trans_y in [False]: + input1_shape = [batch, 64, 350, 75] + input2_shape = [75, 25] + dics = [{ + "trans_x": trans_x, + "trans_y": trans_y, + }] + ops_config = [{ + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"] + }, + "op_outputs": { + "Out": ["output_data"] + }, + "op_attrs": dics[0] + }] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input1_data": + TensorConfig( + data_gen=partial(generate_input, input1_shape)), + "input2_data": + TensorConfig( + data_gen=partial(generate_input, input2_shape)) + }, + outputs=["output_data"]) + + yield program_config def sample_predictor_configs( self, program_config) -> (paddle_infer.Config, List[int], float): @@ -116,8 +116,8 @@ def generate_input(shape): for trans_x in [False]: for trans_y in [False]: - input1_shape = [6, 4] - input2_shape = [4, 4, 9] + input1_shape = [6, 40] + input2_shape = [10, 40, 9] dics = [{ "trans_x": trans_x, "trans_y": trans_y, @@ -155,16 +155,16 @@ def sample_predictor_configs( def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { - "input1_data": [6, 4], - "input2_data": [4, 4, 9] + "input1_data": [6, 40], + "input2_data": [10, 40, 9] } self.dynamic_shape.max_input_shape = { - "input1_data": [6, 4], - "input2_data": [4, 4, 9] + "input1_data": [6, 40], + "input2_data": [10, 40, 9] } self.dynamic_shape.opt_input_shape = { - "input1_data": [6, 4], - "input2_data": [4, 4, 9] + "input1_data": [6, 40], + "input2_data": [10, 40, 9] } attrs = [ From eaa461d8f511fcb6bc4963f3b2aa0b0e469d0b25 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Mon, 19 Sep 2022 09:03:41 +0000 Subject: [PATCH 12/15] commit --- .../inference/test_trt_convert_matmul_v2.py | 84 ++++++++++--------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 9cf769a99161b4..b5d94ebfe3c6fb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -28,6 +28,7 @@ def sample_program_configs(self): def generate_input(shape): return np.random.random(shape).astype(np.float32) + for batch in [10, 11, 12, 13, 14, 15]: for trans_x in [False]: for trans_y in [False]: @@ -114,57 +115,58 @@ def sample_program_configs(self): def generate_input(shape): return np.random.random(shape).astype(np.float32) - for trans_x in [False]: - for trans_y in [False]: - input1_shape = [6, 40] - input2_shape = [10, 40, 9] - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] - ops = self.generate_op_config(ops_config) - - program_config = ProgramConfig( - ops=ops, - weights={}, - inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) - }, - outputs=["output_data"]) - - yield program_config + for batch in [10, 11, 12, 13, 14, 15]: + for trans_x in [False]: + for trans_y in [False]: + input1_shape = [60, 40] + input2_shape = [batch, 40, 90] + dics = [{ + "trans_x": trans_x, + "trans_y": trans_y, + }] + ops_config = [{ + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"] + }, + "op_outputs": { + "Out": ["output_data"] + }, + "op_attrs": dics[0] + }] + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input1_data": + TensorConfig( + data_gen=partial(generate_input, input1_shape)), + "input2_data": + TensorConfig( + data_gen=partial(generate_input, input2_shape)) + }, + outputs=["output_data"]) + + yield program_config def sample_predictor_configs( self, program_config) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { - "input1_data": [6, 40], - "input2_data": [10, 40, 9] + "input1_data": [60, 40], + "input2_data": [10, 40, 90] } self.dynamic_shape.max_input_shape = { - "input1_data": [6, 40], - "input2_data": [10, 40, 9] + "input1_data": [60, 40], + "input2_data": [20, 40, 90] } self.dynamic_shape.opt_input_shape = { - "input1_data": [6, 40], - "input2_data": [10, 40, 9] + "input1_data": [60, 40], + "input2_data": [15, 40, 90] } attrs = [ From 9f12023ab5b35e48bc906028c6dad3fcea737c0d Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Tue, 20 Sep 2022 01:09:43 +0000 Subject: [PATCH 13/15] commit --- paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc index 683c23e9206c3d..327f9bd1e01055 100644 --- a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc @@ -30,7 +30,7 @@ namespace inference { namespace tensorrt { /* - * MatMulOp, IMatrixMultiplyLayer in TRT. This Layer doesn't has weights. + * MatMulV2Op, IMatrixMultiplyLayer in TRT. This Layer doesn't has weights. */ class MatMulV2OpConverter : public OpConverter { public: @@ -105,10 +105,10 @@ class MatMulV2OpConverter : public OpConverter { *input2, matrix_operation_Y); } - VLOG(3) << "Convert a fluid matmul_op_float to TensorRT "; + VLOG(3) << "Convert a fluid matmul_v2_op_float to TensorRT "; RreplenishLayerAndOutput( - layer, "matmul_op_float_no_alpha", {output_name}, test_mode); + layer, "matmul_v2_op_float_no_alpha", {output_name}, test_mode); } }; From 66b91a8ae888a64d1a873fc94f94174df6bc3982 Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Tue, 20 Sep 2022 01:12:20 +0000 Subject: [PATCH 14/15] commit --- paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc index 327f9bd1e01055..8f2d88a95c586a 100644 --- a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc @@ -108,7 +108,7 @@ class MatMulV2OpConverter : public OpConverter { VLOG(3) << "Convert a fluid matmul_v2_op_float to TensorRT "; RreplenishLayerAndOutput( - layer, "matmul_v2_op_float_no_alpha", {output_name}, test_mode); + layer, "matmul_v2_op", {output_name}, test_mode); } }; From e4da56cb2606b04c3fe65a31a149db96af77f32c Mon Sep 17 00:00:00 2001 From: zhoutianzi666 <17801055074@163.com> Date: Tue, 20 Sep 2022 01:18:55 +0000 Subject: [PATCH 15/15] commit --- paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc index 8f2d88a95c586a..c6f5a42a7da891 100644 --- a/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/matmul_v2_op.cc @@ -107,8 +107,7 @@ class MatMulV2OpConverter : public OpConverter { } VLOG(3) << "Convert a fluid matmul_v2_op_float to TensorRT "; - RreplenishLayerAndOutput( - layer, "matmul_v2_op", {output_name}, test_mode); + RreplenishLayerAndOutput(layer, "matmul_v2_op", {output_name}, test_mode); } };