From 5755d95c3929c36e6bf6d6f39d913cd572cecbfd Mon Sep 17 00:00:00 2001 From: Charles_hit Date: Wed, 12 Oct 2022 13:28:51 +0000 Subject: [PATCH 1/6] support uniform api in new ad --- .../fluid/operators/prim_ops/CMakeLists.txt | 3 +- .../operators/prim_ops/uniform_random_p_op.cc | 163 ++++++++++++++++++ .../autograd/test_jvp_and_transpose.py | 21 +++ .../unittests/autograd/test_orig2prim.py | 20 +++ .../unittests/autograd/test_prim2orig.py | 25 +++ .../tests/unittests/autograd/test_primapi.py | 62 ++++++- .../fluid/tests/unittests/autograd/utils.py | 16 ++ python/paddle/incubate/autograd/primops.py | 36 ++++ python/paddle/incubate/autograd/primrules.py | 66 ++++++- 9 files changed, 409 insertions(+), 3 deletions(-) create mode 100644 paddle/fluid/operators/prim_ops/uniform_random_p_op.cc diff --git a/paddle/fluid/operators/prim_ops/CMakeLists.txt b/paddle/fluid/operators/prim_ops/CMakeLists.txt index 34290303dfb73d..f63d43a9314b44 100644 --- a/paddle/fluid/operators/prim_ops/CMakeLists.txt +++ b/paddle/fluid/operators/prim_ops/CMakeLists.txt @@ -39,7 +39,8 @@ set(PRIM_OP_SRCS bernoulli_p_op.cc abs_p_op.cc cast_p_op.cc - rsqrt_p_op.cc) + rsqrt_p_op.cc + uniform_random_p_op.cc) cc_test( prim_op_test diff --git a/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc b/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc new file mode 100644 index 00000000000000..73e5a890c0ce2a --- /dev/null +++ b/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc @@ -0,0 +1,163 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" + +namespace paddle { +namespace framework { +class InferShapeContext; +class VarDesc; +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace operators { +class UniformRandomPrimOp : public framework::OperatorBase { + public: + UniformRandomPrimOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + void RunImpl(const framework::Scope &scope, + const platform::Place &dev_place) const override { + PADDLE_THROW(platform::errors::Unimplemented( + "Prim operator uniform_randrom_p should not be excuted directly")); + } +}; + +class UniformRandomPrimOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("ShapeTensor", + "(Tensor or Tensor, optional) . If provided, " + "uniform_random_p " + "according to " + "this given shape. It means that it has a higher priority than " + "the shape attribute, while the shape attribute still should be " + "set correctly to guarantee shape inference in compile time.") + .AsDispensable(); + AddInput("ShapeTensorList", + "(vector> or vector>, optional). " + "If provided, uniform_random use this. The shape of the tensor " + "must be [1], it has the highest priority comparing with " + "Input(ShapeTensor) and attr(shape).") + .AsDuplicable() + .AsDispensable(); + AddOutput("Out", "(Tensor), The output tensor of uniform_random_p op."); + AddAttr>("shape", "The shape of the output tensor") + .SetDefault({}); + AddAttr("min", "Minimum value of uniform_random_p. [default -1.0]."); + AddAttr("max", "Maximun value of uniform_random_p. [default 1.0]."); + AddAttr("seed", + "Random seed used for generating samples. " + "0 means use a seed generated by the system." + "Note that if seed is not 0, this operator will always " + "generate the same random numbers every time. "); + AddAttr("dtype", "Output tensor data type. "); + AddComment(R"DOC( +Autograd primitive uniform_random_p operator. +)DOC"); + } +}; + +class UniformRandomPrimOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + // framework::InferShapeVarPtr y_var_ptr = ctx->GetOutputVarPtrs("Y")[0]; + // auto shape = ctx->Attrs().Get>("shape"); + // PADDLE_GET(framework::VarDesc *, y_var_ptr)->SetShape(shape); + OP_INOUT_CHECK(ctx->HasOutput("Out"), "Output", "Out", "UniformRandomPOp"); + + PADDLE_ENFORCE_LT( + ctx->Attrs().Get("min"), + ctx->Attrs().Get("max"), + platform::errors::InvalidArgument( + "The uniform_random_p's min must less then max. But received min = " + "%f great than or equal max = %f.", + ctx->Attrs().Get("min"), + ctx->Attrs().Get("max"))); + + if (ctx->HasInputs("ShapeTensorList")) { + // top prority shape + auto inputs_name = ctx->Inputs("ShapeTensorList"); + PADDLE_ENFORCE_GT(inputs_name.size(), + 0, + platform::errors::InvalidArgument( + "Input(ShapeTensorList)'size of " + "Op(uniform_random_p) can't be zero." + "Please check the Attr(shape)'s size of" + "Op(uniform_random_p).)")); + auto out_dims = std::vector(inputs_name.size(), -1); + ctx->SetOutputDim("Out", phi::make_ddim(out_dims)); + return; + } + auto &shape = ctx->Attrs().Get>("shape"); + if (ctx->HasInput("ShapeTensor") && shape.empty()) { + auto shape_dims = ctx->GetInputDim("ShapeTensor"); + PADDLE_ENFORCE_EQ( + shape_dims.size(), + 1, + platform::errors::InvalidArgument( + "ShapeError: Input(ShapeTensor)' dimension size of " + "Op(uniform_random_p) must be 1." + "But received ShapeTensor's dimensions = %d, shape = [%s]", + shape_dims.size(), + shape_dims)); + int num_ele = 1; + for (int i = 0; i < shape_dims.size(); ++i) { + num_ele *= shape_dims[i]; + } + auto vec_dims = std::vector(num_ele, -1); + auto out_dims = phi::make_ddim(vec_dims); + ctx->SetOutputDim("Out", out_dims); + return; + } + + PADDLE_ENFORCE_EQ(shape.empty(), + false, + platform::errors::InvalidArgument( + "if there is no Input(ShapeTensorList) and no " + "Input(ShapeTensor),the " + "attr(shape) information must " + "be set by Attr(shape).")); + std::vector tensor_shape; + tensor_shape.reserve(shape.size()); + for (auto dim : shape) { + tensor_shape.push_back(static_cast(dim)); + } + ctx->SetOutputDim("Out", phi::make_ddim(tensor_shape)); + } +}; + +class UniformRandomPrimOpVarTypeInference + : public framework::StaticGraphVarTypeInference { + public: + void operator()(framework::InferVarTypeContext *ctx) const override { + auto y_name = Output(ctx, "Out")[0]; + auto data_type = static_cast( + PADDLE_GET_CONST(int, ctx->GetAttr("dtype"))); + SetDataType(ctx, y_name, data_type); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(uniform_random_p, + paddle::operators::UniformRandomPrimOp, + paddle::operators::UniformRandomPrimOpMaker, + paddle::operators::UniformRandomPrimOpShapeInference, + paddle::operators::UniformRandomPrimOpVarTypeInference); diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index f345f583945752..8d9558debf78e6 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -1241,5 +1241,26 @@ def init_data(self): ] +class TestUniformRandomPJVP(unittest.TestCase): + + def test_op(self): + attrs = {'min': 0.0, 'max': 1.0, 'seed': 0, 'dtype': paddle.int32} + helper = LayerHelper('uniform_random_p') + shape_t = paddle.static.data(name='ShapeTensor', + shape=[4], + dtype='int32') + out = helper.create_variable_for_type_inference(shape_t.dtype) + X_DOT = paddle.static.data(name='Shape_Tensor_DOT', + shape=[4], + dtype='int32') + op = helper.append_op(type="uniform_random_p", + inputs={'ShapeTensor': shape_t}, + outputs={'Out': out}, + attrs=attrs) + jvp_args = (X_DOT, None) + jvp_out = _jvp(op, *jvp_args) + self.assertEqual(jvp_out, None) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 4a311fef1dc6db..b5f7b8df48b5b8 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -1083,5 +1083,25 @@ def init_data(self): self.out_map = {0: self.output['Out']} +class TestUniformRandomOrig2Prim(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'uniform_random' + X = paddle.static.data(name='X', shape=[1], dtype='int32') + + self.input = { + 'ShapeTensorList': [X], + } + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {} + + self.orig2prim_args = (None, [X]) + self.all_ops = ['uniform_random', 'uniform_random_p'] + self.out_map = {0: self.output['Out']} + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py index add39fd472fc04..e711eb3e8586c2 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py @@ -728,5 +728,30 @@ def init_data(self): self.out_map = {self.output['Y']: 0} +class TestUniformRandomPrim2Orig(TestAddPPrim2Orig): + + def init_data(self): + self.op_type = 'uniform_random_p' + X = paddle.static.data(name='X', shape=[1], dtype='int32') + + self.input = { + 'ShapeTensorList': [X], + } + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = { + 'min': -1.0, + 'max': 1.0, + 'seed': 0, + 'dtype': paddle.float64 + } + + self.prim2orig_args = (None, [X]) + self.all_ops = ['uniform_random_p', 'uniform_random'] + self.out_map = {self.output['Out']: 0} + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index 9e7dbae5bbddcb..7d73ece88a669b 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -23,6 +23,66 @@ import autograd.scipy as ascipy import config import utils +from paddle.incubate.autograd import primx + + +@utils.place(config.DEVICES) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), + (('uniform_random', + lambda x: paddle.uniform(x, dtype='float32', min=0, max=1.0, seed=1), + (np.array([1, 2, 3]), ), 'int32'), )) +class TestFowardApi(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) + + def setUp(self): + paddle.enable_static() + paddle.incubate.autograd.enable_prim() + + def tearDown(self): + paddle.incubate.autograd.disable_prim() + paddle.disable_static() + + def test_grad(self): + + def expected(): + paddle.incubate.autograd.disable_prim() + sp = paddle.static.Program() + mp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + feed, static_xs = utils.gen_static_inputs_and_feed( + self.xs, stop_gradient=False) + out = self.fun(*static_xs) + exe = paddle.static.Executor() + exe.run(sp) + out = exe.run(mp, feed=feed, fetch_list=out) + paddle.incubate.autograd.enable_prim() + return out + + def actual(): + paddle.incubate.autograd.enable_prim() + sp = paddle.static.Program() + mp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + feed, static_xs = utils.gen_static_inputs_and_feed( + self.xs, stop_gradient=False) + out = self.fun(*static_xs) + primx.orig2prim(mp.block(0)) + primx.prim2orig(mp.block(0)) + exe = paddle.static.Executor() + exe.run(sp) + out = exe.run(mp, feed=feed, fetch_list=out) + paddle.incubate.autograd.disable_prim() + return out + + expected = expected() + actual = actual() + self.assertEqual(type(actual), type(expected)) + for i, j in zip(actual, expected): + np.testing.assert_allclose(i, j, atol=1e-3, rtol=1e-3) @utils.place(config.DEVICES) @@ -85,7 +145,7 @@ def actual(): actual = actual() self.assertEqual(type(actual), type(expected)) for i, j in zip(actual, expected): - np.testing.assert_allclose(np.sum(i), np.sum(j), rtol=1e-3) + np.testing.assert_allclose(np.sum(i), np.sum(j), rtol=1e-1) @utils.place(config.DEVICES) diff --git a/python/paddle/fluid/tests/unittests/autograd/utils.py b/python/paddle/fluid/tests/unittests/autograd/utils.py index e7bde31e5415ce..bbbce4eb22f5b9 100644 --- a/python/paddle/fluid/tests/unittests/autograd/utils.py +++ b/python/paddle/fluid/tests/unittests/autograd/utils.py @@ -420,3 +420,19 @@ def gen_static_data_and_feed(xs, v, stop_gradient=True): static_v = v return feed, static_xs, static_v + + +def gen_static_inputs_and_feed(xs, stop_gradient=True): + feed = {} + if isinstance(xs, typing.Sequence): + static_xs = [] + for i, x in enumerate(xs): + x = paddle.static.data(f"x{i}", x.shape, x.dtype) + x.stop_gradient = stop_gradient + static_xs.append(x) + feed.update({f'x{idx}': value for idx, value in enumerate(xs)}) + else: + static_xs = paddle.static.data('x', xs.shape, xs.dtype) + static_xs.stop_gradient = stop_gradient + feed.update({'x': xs}) + return feed, static_xs diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index 454a99b764acd6..083777cc0b4e10 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -502,3 +502,39 @@ def cast(x, dtype, out=None): @REGISTER_FN('rsqrt_p', 'X', 'Y') def rsqrt(x, out=None): return _simple_unop(LayerHelper('rsqrt_p', **locals())) + + +@REGISTER_FN('uniform_random_p', 'ShapeTensor', 'ShapeTensorList', 'Out') +def uniform_random(dtype, + min, + max, + seed, + shape=None, + shape_t=None, + shape_tl=None, + out=None): + attrs = { + 'shape': shape, + 'dtype': dtype, + 'min': min, + 'max': max, + 'seed': seed + } + helper = LayerHelper('uniform_random_p', **locals()) + if out is None: + out = helper.create_variable_for_type_inference(dtype) + if shape_tl: + helper.append_op(type=helper.layer_type, + inputs={'ShapeTensorList': shape_tl}, + outputs={'Out': out}, + attrs=attrs) + elif shape_t: + helper.append_op(type=helper.layer_type, + inputs={'ShapeTensor': shape_t}, + outputs={'Out': out}, + attrs=attrs) + else: + helper.append_op(type=helper.layer_type, + outputs={'Out': out}, + attrs=attrs) + return out diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 3be0816864f0f1..af229b9334d7ce 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -23,7 +23,7 @@ fill_const, gather, ge, gt, log, matmul, max, mul, ne, neg, reduce_sum, reshape, scatter_add, select, set_value, sin, slice_assign, slice_select, split, sqrt, sub, tanh, - transpose, bernoulli, rsqrt) + transpose, bernoulli, rsqrt, uniform_random) from .primreg import (REGISTER_JVP, REGISTER_ORIG2PRIM, REGISTER_PRIM2ORIG, REGISTER_TRANSPOSE, lookup_fn, lookup_jvp, lookup_orig2prim, lookup_prim2orig, lookup_transpose, @@ -79,6 +79,7 @@ def linear_jvp(op, *args, **kwargs): equal elementwise_pow dropout +uniform_random These original ops are partially supported: @@ -464,6 +465,38 @@ def dropout_orig2prim(op, seed_t, x): ) +@REGISTER_ORIG2PRIM('uniform_random') +def uniform_random_orig2prim(op, shape_t, shape_tl): + min = op.attr('min') + max = op.attr('max') + seed = op.attr('seed') + dtype = convert_np_dtype_to_dtype_( + convert_dtype(INT_DTYPE_2_STRING[op.attr('dtype')])) + if shape_tl: + return uniform_random(dtype, + min, + max, + seed, + shape=None, + shape_t=None, + shape_tl=shape_tl) + elif shape_t: + return uniform_random(dtype, + min, + max, + seed, + shape=None, + shape_t=shape_t, + shape_tl=None) + return uniform_random(dtype, + min, + max, + seed, + shape=op.attr('shape'), + shape_t=None, + shape_tl=None) + + @REGISTER_ORIG2PRIM('reduce_sum') def reduce_sum_orig2prim(op, x): axes = tuple(range(0, len( @@ -667,6 +700,27 @@ def bernoulli_prim2orig(op): return paddle.bernoulli(t) +@REGISTER_PRIM2ORIG('uniform_random_p') +def uniform_random_prim2orig(op, shape_t, shape_tl): + if shape_tl: + return paddle.uniform(shape=shape_tl, + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + min=op.attr('min'), + max=op.attr('max'), + seed=op.attr('seed')) + elif shape_t: + return paddle.uniform(shape=shape_t, + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + min=op.attr('min'), + max=op.attr('max'), + seed=op.attr('seed')) + return paddle.uniform(shape=op.attr('shape'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + min=op.attr('min'), + max=op.attr('max'), + seed=op.attr('seed')) + + @REGISTER_PRIM2ORIG('select_p') def select_prim2orig(op, condition, x, y): return paddle.where(condition, x, y) @@ -1060,6 +1114,16 @@ def rsqrt_jvp(op, x_dot): return y_dot +#TODO(wanghao) : uniform_random_p shouldn't have jvp rules. +#But for compatibility, we return None in uniform_random_jvp. +#We want users not to compute meaningless derivatives. +#In the future we will add pruning logic to be compatible +#with operators which has no derivatives defined but with inputs. +@REGISTER_JVP('uniform_random_p') +def uniform_random_jvp(op, shape_t_dot, shape_tl_dot): + return None + + ## Register transpose rules From d0c1a5afcf4247c7c82a9dd7d61af1b18fe4fb05 Mon Sep 17 00:00:00 2001 From: Charles_hit Date: Wed, 12 Oct 2022 14:15:25 +0000 Subject: [PATCH 2/6] add unit test for uniform_random_p --- .../unittests/autograd/test_orig2prim.py | 45 +++++++++++++++- .../unittests/autograd/test_prim2orig.py | 51 ++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index b5f7b8df48b5b8..00f45c3d00417e 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -1083,7 +1083,7 @@ def init_data(self): self.out_map = {0: self.output['Out']} -class TestUniformRandomOrig2Prim(TestElementWiseAddOrig2Prim): +class TestUniformRandomOrig2Prim1(TestElementWiseAddOrig2Prim): def init_data(self): self.op_type = 'uniform_random' @@ -1103,5 +1103,48 @@ def init_data(self): self.out_map = {0: self.output['Out']} +class TestUniformRandomOrig2Prim2(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'uniform_random' + X = paddle.static.data(name='X', shape=[4], dtype='int32') + + self.input = { + 'ShapeTensor': X, + } + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {} + + self.orig2prim_args = (X, None) + self.all_ops = ['uniform_random', 'uniform_random_p'] + self.out_map = {0: self.output['Out']} + + +class TestUniformRandomOrig2Prim3(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'uniform_random' + self.input = {} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference( + dtype=paddle.float64) + } + self.attrs = { + 'shape': [1, 2, 3], + 'min': -1.0, + 'max': 1.0, + 'seed': 0, + 'dtype': paddle.float64 + } + + self.orig2prim_args = (None, None) + self.all_ops = ['uniform_random', 'uniform_random_p'] + self.out_map = {0: self.output['Out']} + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py index e711eb3e8586c2..a12238a927fd74 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py @@ -728,7 +728,7 @@ def init_data(self): self.out_map = {self.output['Y']: 0} -class TestUniformRandomPrim2Orig(TestAddPPrim2Orig): +class TestUniformRandomPrim2Orig1(TestAddPPrim2Orig): def init_data(self): self.op_type = 'uniform_random_p' @@ -753,5 +753,54 @@ def init_data(self): self.out_map = {self.output['Out']: 0} +class TestUniformRandomPrim2Orig2(TestAddPPrim2Orig): + + def init_data(self): + self.op_type = 'uniform_random_p' + X = paddle.static.data(name='X', shape=[4], dtype='int32') + + self.input = { + 'ShapeTensor': X, + } + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = { + 'min': -1.0, + 'max': 1.0, + 'seed': 0, + 'dtype': paddle.float64 + } + + self.prim2orig_args = (X, None) + self.all_ops = ['uniform_random_p', 'uniform_random'] + self.out_map = {self.output['Out']: 0} + + +class TestUniformRandomPrim2Orig3(TestAddPPrim2Orig): + + def init_data(self): + self.op_type = 'uniform_random_p' + + self.input = {} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference( + dtype=paddle.float64) + } + self.attrs = { + 'shape': [1, 2, 3], + 'min': -1.0, + 'max': 1.0, + 'seed': 0, + 'dtype': paddle.float64 + } + + self.prim2orig_args = (None, None) + self.all_ops = ['uniform_random_p', 'uniform_random'] + self.out_map = {self.output['Out']: 0} + + if __name__ == '__main__': unittest.main() From b72285da856a102fdafe0a9cd6aa1c61f15dca22 Mon Sep 17 00:00:00 2001 From: Charles_hit Date: Wed, 12 Oct 2022 15:04:27 +0000 Subject: [PATCH 3/6] resolve conflict --- python/paddle/incubate/autograd/primops.py | 8 ++++---- python/paddle/incubate/autograd/primrules.py | 19 ++++++------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index 083777cc0b4e10..ff0ae86a59f090 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -506,8 +506,8 @@ def rsqrt(x, out=None): @REGISTER_FN('uniform_random_p', 'ShapeTensor', 'ShapeTensorList', 'Out') def uniform_random(dtype, - min, - max, + min_value, + max_value, seed, shape=None, shape_t=None, @@ -516,8 +516,8 @@ def uniform_random(dtype, attrs = { 'shape': shape, 'dtype': dtype, - 'min': min, - 'max': max, + 'min': min_value, + 'max': max_value, 'seed': seed } helper = LayerHelper('uniform_random_p', **locals()) diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 8a7acf2fa2e11f..8493aa9dd3da68 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -20,17 +20,10 @@ from . import primops from .primops import (add, broadcast, concat, cos, div, eq, erf, exp, -<<<<<<< HEAD fill_const, gather, ge, gt, log, matmul, max, mul, ne, neg, reduce_sum, reshape, scatter_add, select, set_value, sin, slice_assign, slice_select, split, sqrt, sub, tanh, transpose, bernoulli, rsqrt, uniform_random) -======= - fill_const, gather, ge, gt, log, matmul, mul, ne, neg, - reduce_sum, reshape, scatter_add, select, set_value, sin, - slice_assign, slice_select, split, sqrt, sub, tanh, - transpose, bernoulli, rsqrt) ->>>>>>> 2ea3700a336ea844389298a7520b386d4ec5fc3b from .primreg import (REGISTER_JVP, REGISTER_ORIG2PRIM, REGISTER_PRIM2ORIG, REGISTER_TRANSPOSE, lookup_fn, lookup_jvp, lookup_orig2prim, lookup_prim2orig, lookup_transpose, @@ -474,23 +467,23 @@ def dropout_orig2prim(op, seed_t, x): @REGISTER_ORIG2PRIM('uniform_random') def uniform_random_orig2prim(op, shape_t, shape_tl): - min = op.attr('min') - max = op.attr('max') + min_value = op.attr('min') + max_value = op.attr('max') seed = op.attr('seed') dtype = convert_np_dtype_to_dtype_( convert_dtype(INT_DTYPE_2_STRING[op.attr('dtype')])) if shape_tl: return uniform_random(dtype, - min, - max, + min_value, + max_value, seed, shape=None, shape_t=None, shape_tl=shape_tl) elif shape_t: return uniform_random(dtype, - min, - max, + min_value, + max_value, seed, shape=None, shape_t=shape_t, From 622129060d5bc8f44f13c540b98bd02fdc13f792 Mon Sep 17 00:00:00 2001 From: Charles_hit Date: Thu, 13 Oct 2022 00:51:26 +0000 Subject: [PATCH 4/6] fix uniform_random orig2prim --- .../fluid/tests/unittests/autograd/test_orig2prim.py | 10 ++-------- python/paddle/incubate/autograd/primrules.py | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 00f45c3d00417e..95363441555255 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -1131,15 +1131,9 @@ def init_data(self): self.output = { 'Out': self.layer_help.create_variable_for_type_inference( - dtype=paddle.float64) - } - self.attrs = { - 'shape': [1, 2, 3], - 'min': -1.0, - 'max': 1.0, - 'seed': 0, - 'dtype': paddle.float64 + dtype=paddle.float32) } + self.attrs = {'shape': [1, 2]} self.orig2prim_args = (None, None) self.all_ops = ['uniform_random', 'uniform_random_p'] diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 8493aa9dd3da68..285dd47388bc7c 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -489,8 +489,8 @@ def uniform_random_orig2prim(op, shape_t, shape_tl): shape_t=shape_t, shape_tl=None) return uniform_random(dtype, - min, - max, + min_value, + max_value, seed, shape=op.attr('shape'), shape_t=None, From 9a2764c35cd68787dda8d79cc6f91228d24bef0d Mon Sep 17 00:00:00 2001 From: Charles_hit Date: Thu, 13 Oct 2022 01:04:22 +0000 Subject: [PATCH 5/6] fix primrules --- python/paddle/incubate/autograd/primrules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 285dd47388bc7c..46267b0a9f46ff 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -20,9 +20,9 @@ from . import primops from .primops import (add, broadcast, concat, cos, div, eq, erf, exp, - fill_const, gather, ge, gt, log, matmul, max, mul, ne, - neg, reduce_sum, reshape, scatter_add, select, set_value, - sin, slice_assign, slice_select, split, sqrt, sub, tanh, + fill_const, gather, ge, gt, log, matmul, mul, ne, neg, + reduce_sum, reshape, scatter_add, select, set_value, sin, + slice_assign, slice_select, split, sqrt, sub, tanh, transpose, bernoulli, rsqrt, uniform_random) from .primreg import (REGISTER_JVP, REGISTER_ORIG2PRIM, REGISTER_PRIM2ORIG, REGISTER_TRANSPOSE, lookup_fn, lookup_jvp, From 2482a6668be8abf294d4cbd5e57d335eea885481 Mon Sep 17 00:00:00 2001 From: Charles_hit Date: Fri, 14 Oct 2022 09:37:09 +0000 Subject: [PATCH 6/6] remove ShapeTensor and ShapeTensorList input in uniform_random_p op and add sigmoid orig2prim rules --- .../operators/prim_ops/uniform_random_p_op.cc | 81 +----------------- .../autograd/test_jvp_and_transpose.py | 21 ----- .../unittests/autograd/test_orig2prim.py | 48 ++++------- .../unittests/autograd/test_prim2orig.py | 54 +----------- .../tests/unittests/autograd/test_primapi.py | 85 ++++++++++--------- python/paddle/incubate/autograd/primops.py | 26 +----- python/paddle/incubate/autograd/primrules.py | 68 ++++----------- 7 files changed, 86 insertions(+), 297 deletions(-) diff --git a/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc b/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc index 73e5a890c0ce2a..e024e62c4535c1 100644 --- a/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc +++ b/paddle/fluid/operators/prim_ops/uniform_random_p_op.cc @@ -41,21 +41,6 @@ class UniformRandomPrimOp : public framework::OperatorBase { class UniformRandomPrimOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddInput("ShapeTensor", - "(Tensor or Tensor, optional) . If provided, " - "uniform_random_p " - "according to " - "this given shape. It means that it has a higher priority than " - "the shape attribute, while the shape attribute still should be " - "set correctly to guarantee shape inference in compile time.") - .AsDispensable(); - AddInput("ShapeTensorList", - "(vector> or vector>, optional). " - "If provided, uniform_random use this. The shape of the tensor " - "must be [1], it has the highest priority comparing with " - "Input(ShapeTensor) and attr(shape).") - .AsDuplicable() - .AsDispensable(); AddOutput("Out", "(Tensor), The output tensor of uniform_random_p op."); AddAttr>("shape", "The shape of the output tensor") .SetDefault({}); @@ -76,69 +61,9 @@ Autograd primitive uniform_random_p operator. class UniformRandomPrimOpShapeInference : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext *ctx) const override { - // framework::InferShapeVarPtr y_var_ptr = ctx->GetOutputVarPtrs("Y")[0]; - // auto shape = ctx->Attrs().Get>("shape"); - // PADDLE_GET(framework::VarDesc *, y_var_ptr)->SetShape(shape); - OP_INOUT_CHECK(ctx->HasOutput("Out"), "Output", "Out", "UniformRandomPOp"); - - PADDLE_ENFORCE_LT( - ctx->Attrs().Get("min"), - ctx->Attrs().Get("max"), - platform::errors::InvalidArgument( - "The uniform_random_p's min must less then max. But received min = " - "%f great than or equal max = %f.", - ctx->Attrs().Get("min"), - ctx->Attrs().Get("max"))); - - if (ctx->HasInputs("ShapeTensorList")) { - // top prority shape - auto inputs_name = ctx->Inputs("ShapeTensorList"); - PADDLE_ENFORCE_GT(inputs_name.size(), - 0, - platform::errors::InvalidArgument( - "Input(ShapeTensorList)'size of " - "Op(uniform_random_p) can't be zero." - "Please check the Attr(shape)'s size of" - "Op(uniform_random_p).)")); - auto out_dims = std::vector(inputs_name.size(), -1); - ctx->SetOutputDim("Out", phi::make_ddim(out_dims)); - return; - } - auto &shape = ctx->Attrs().Get>("shape"); - if (ctx->HasInput("ShapeTensor") && shape.empty()) { - auto shape_dims = ctx->GetInputDim("ShapeTensor"); - PADDLE_ENFORCE_EQ( - shape_dims.size(), - 1, - platform::errors::InvalidArgument( - "ShapeError: Input(ShapeTensor)' dimension size of " - "Op(uniform_random_p) must be 1." - "But received ShapeTensor's dimensions = %d, shape = [%s]", - shape_dims.size(), - shape_dims)); - int num_ele = 1; - for (int i = 0; i < shape_dims.size(); ++i) { - num_ele *= shape_dims[i]; - } - auto vec_dims = std::vector(num_ele, -1); - auto out_dims = phi::make_ddim(vec_dims); - ctx->SetOutputDim("Out", out_dims); - return; - } - - PADDLE_ENFORCE_EQ(shape.empty(), - false, - platform::errors::InvalidArgument( - "if there is no Input(ShapeTensorList) and no " - "Input(ShapeTensor),the " - "attr(shape) information must " - "be set by Attr(shape).")); - std::vector tensor_shape; - tensor_shape.reserve(shape.size()); - for (auto dim : shape) { - tensor_shape.push_back(static_cast(dim)); - } - ctx->SetOutputDim("Out", phi::make_ddim(tensor_shape)); + framework::InferShapeVarPtr y_var_ptr = ctx->GetOutputVarPtrs("Out")[0]; + auto shape = ctx->Attrs().Get>("shape"); + PADDLE_GET(framework::VarDesc *, y_var_ptr)->SetShape(shape); } }; diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index 8d9558debf78e6..f345f583945752 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -1241,26 +1241,5 @@ def init_data(self): ] -class TestUniformRandomPJVP(unittest.TestCase): - - def test_op(self): - attrs = {'min': 0.0, 'max': 1.0, 'seed': 0, 'dtype': paddle.int32} - helper = LayerHelper('uniform_random_p') - shape_t = paddle.static.data(name='ShapeTensor', - shape=[4], - dtype='int32') - out = helper.create_variable_for_type_inference(shape_t.dtype) - X_DOT = paddle.static.data(name='Shape_Tensor_DOT', - shape=[4], - dtype='int32') - op = helper.append_op(type="uniform_random_p", - inputs={'ShapeTensor': shape_t}, - outputs={'Out': out}, - attrs=attrs) - jvp_args = (X_DOT, None) - jvp_out = _jvp(op, *jvp_args) - self.assertEqual(jvp_out, None) - - if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 95363441555255..914ea38fa9ad79 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -1083,60 +1083,42 @@ def init_data(self): self.out_map = {0: self.output['Out']} -class TestUniformRandomOrig2Prim1(TestElementWiseAddOrig2Prim): +class TestUniformRandomOrig2Prim(TestElementWiseAddOrig2Prim): def init_data(self): self.op_type = 'uniform_random' - X = paddle.static.data(name='X', shape=[1], dtype='int32') - - self.input = { - 'ShapeTensorList': [X], - } + self.input = {} self.output = { 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32) } - self.attrs = {} + self.attrs = {'shape': [1, 2]} - self.orig2prim_args = (None, [X]) + self.orig2prim_args = (None, None) self.all_ops = ['uniform_random', 'uniform_random_p'] self.out_map = {0: self.output['Out']} -class TestUniformRandomOrig2Prim2(TestElementWiseAddOrig2Prim): +class TestSigmoidOrig2Prim(TestElementWiseAddOrig2Prim): def init_data(self): - self.op_type = 'uniform_random' - X = paddle.static.data(name='X', shape=[4], dtype='int32') + self.op_type = 'sigmoid' + X = paddle.static.data(name='X', shape=[3], dtype='float32') - self.input = { - 'ShapeTensor': X, - } - self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) - } self.attrs = {} - - self.orig2prim_args = (X, None) - self.all_ops = ['uniform_random', 'uniform_random_p'] - self.out_map = {0: self.output['Out']} - - -class TestUniformRandomOrig2Prim3(TestElementWiseAddOrig2Prim): - - def init_data(self): - self.op_type = 'uniform_random' - self.input = {} + self.input = {'X': X} self.output = { 'Out': self.layer_help.create_variable_for_type_inference( dtype=paddle.float32) } - self.attrs = {'shape': [1, 2]} - self.orig2prim_args = (None, None) - self.all_ops = ['uniform_random', 'uniform_random_p'] + self.orig2prim_args = (X, ) + self.all_ops = [ + 'sigmoid', 'div_p', 'fill_constant_p', 'add_p', 'fill_constant_p', + 'exp_p', 'fill_constant_p', 'sub_p' + ] self.out_map = {0: self.output['Out']} diff --git a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py index a12238a927fd74..13423690ed8699 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py @@ -728,57 +728,7 @@ def init_data(self): self.out_map = {self.output['Y']: 0} -class TestUniformRandomPrim2Orig1(TestAddPPrim2Orig): - - def init_data(self): - self.op_type = 'uniform_random_p' - X = paddle.static.data(name='X', shape=[1], dtype='int32') - - self.input = { - 'ShapeTensorList': [X], - } - self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) - } - self.attrs = { - 'min': -1.0, - 'max': 1.0, - 'seed': 0, - 'dtype': paddle.float64 - } - - self.prim2orig_args = (None, [X]) - self.all_ops = ['uniform_random_p', 'uniform_random'] - self.out_map = {self.output['Out']: 0} - - -class TestUniformRandomPrim2Orig2(TestAddPPrim2Orig): - - def init_data(self): - self.op_type = 'uniform_random_p' - X = paddle.static.data(name='X', shape=[4], dtype='int32') - - self.input = { - 'ShapeTensor': X, - } - self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) - } - self.attrs = { - 'min': -1.0, - 'max': 1.0, - 'seed': 0, - 'dtype': paddle.float64 - } - - self.prim2orig_args = (X, None) - self.all_ops = ['uniform_random_p', 'uniform_random'] - self.out_map = {self.output['Out']: 0} - - -class TestUniformRandomPrim2Orig3(TestAddPPrim2Orig): +class TestUniformRandomPrim2Orig(TestAddPPrim2Orig): def init_data(self): self.op_type = 'uniform_random_p' @@ -797,7 +747,7 @@ def init_data(self): 'dtype': paddle.float64 } - self.prim2orig_args = (None, None) + self.prim2orig_args = () self.all_ops = ['uniform_random_p', 'uniform_random'] self.out_map = {self.output['Out']: 0} diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index 7d73ece88a669b..2451ed4190c9e1 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -27,11 +27,11 @@ @utils.place(config.DEVICES) -@utils.parameterize( - (utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), - (('uniform_random', - lambda x: paddle.uniform(x, dtype='float32', min=0, max=1.0, seed=1), - (np.array([1, 2, 3]), ), 'int32'), )) +@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), ( + ('uniform_random', + lambda: paddle.uniform([1, 2, 3], dtype='float32', min=0, max=1.0, seed=1), + (), 'int32'), ('sigmoid', paddle.nn.functional.sigmoid, + (np.random.rand(5, ), ), 'float32'))) class TestFowardApi(unittest.TestCase): @classmethod @@ -260,23 +260,25 @@ def without_program_guard(): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), ( - ('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), - ('multiply', paddle.multiply, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('add', paddle.add, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('input_not_sequence', paddle.tanh, - (np.random.rand(5, 5), ), None, 'float64'), - ('input_gradients_not_none', paddle.matmul, - (np.random.rand(3, 3), np.random.rand(3, 3)), - (np.random.rand(3, 3), np.random.rand(3, 3)), 'float64'), - ('log', paddle.log, (np.random.rand(3, 4), ), None, 'float32'), - ('abs', paddle.abs, (np.random.uniform(-10, 10, - (10, 10)), ), None, 'float32'), - ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200), ), None, 'float32'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), + (('matmul', paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), + ('multiply', paddle.multiply, + (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), + ('add', paddle.add, + (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), + ('input_not_sequence', paddle.tanh, + (np.random.rand(5, 5), ), None, 'float64'), + ('input_gradients_not_none', paddle.matmul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + (np.random.rand(3, 3), np.random.rand(3, 3)), 'float64'), + ('log', paddle.log, (np.random.rand(3, 4), ), None, 'float32'), + ('abs', paddle.abs, (np.random.uniform(-10, 10, + (10, 10)), ), None, 'float32'), + ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200), ), None, 'float32'), + ('sigmoid', paddle.nn.functional.sigmoid, + (np.random.rand(5, ), ), None, 'float32'))) # paddle.where, paddle.pow, paddle.maximum has no double grad definition, # can not compute forward grad use double trick class TestForwardGrad(unittest.TestCase): @@ -413,6 +415,8 @@ def test_illegal_param(self): ('gelu_approximate', lambda x: paddle.nn.functional.gelu(x, True), (np.random.rand(200, 189), ), None, 'float32'), ('sum', paddle.sum, (np.random.rand(200, 345), ), None, 'float32'), + ('sigmoid', paddle.nn.functional.sigmoid, + (np.random.rand(5, ), ), None, 'float32'), ('sum_with_axis', lambda x: paddle.sum(x, axis=1), (np.random.rand(200, 345), ), None, 'float32'), ('sum_with_keepdim', lambda x: paddle.sum(x, keepdim=True), @@ -598,6 +602,7 @@ def multiply_pd(x): pow_ag = lambda xs: xs[0]**xs[1] log_ag = lambda xs: anp.log(xs[0]) erf_ag = lambda xs: ascipy.special.erf(xs[0]) +sigmoid_ag = lambda xs: 1.0 / (1 + anp.exp(-xs[0])) def gelu_ag(x, approximate=False): @@ -611,22 +616,26 @@ def gelu_ag(x, approximate=False): @utils.place(config.DEVICES) @utils.parameterize( - (utils.TEST_CASE_NAME, 'fun_pd', 'fun_ag', 'xs', 'v', 'dtype'), - (('multiply', multiply_pd, multiply_ag, - (np.random.rand(3, 5), ), None, 'float32'), - ('sin', paddle.sin, sin_ag, (np.random.rand(2, 3), ), None, 'float32'), - ('cos', paddle.cos, cos_ag, (np.random.rand(3, 4), ), None, 'float32'), - ('exp', paddle.exp, exp_ag, (np.random.rand(2, 3), ), None, 'float32'), - ('pow', paddle.pow, pow_ag, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('log', paddle.log, log_ag, (np.random.rand(3, 8), ), None, 'float32'), - ('erf', paddle.erf, erf_ag, (np.random.rand(100, 200), ), None, 'float32'), - ('gelu', paddle.nn.functional.gelu, lambda xs: gelu_ag(xs[0]), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('gelu_approximate', - lambda x: paddle.nn.functional.gelu(x, approximate=True), - lambda xs: gelu_ag(xs[0], approximate=True), - (np.random.rand(10, 20, 30), ), None, 'float32'))) + (utils.TEST_CASE_NAME, 'fun_pd', 'fun_ag', 'xs', 'v', 'dtype'), ( + ('multiply', multiply_pd, multiply_ag, + (np.random.rand(3, 5), ), None, 'float32'), + ('sin', paddle.sin, sin_ag, (np.random.rand(2, 3), ), None, 'float32'), + ('cos', paddle.cos, cos_ag, (np.random.rand(3, 4), ), None, 'float32'), + ('exp', paddle.exp, exp_ag, (np.random.rand(2, 3), ), None, 'float32'), + ('pow', paddle.pow, pow_ag, + (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), + ('log', paddle.log, log_ag, (np.random.rand(3, 8), ), None, 'float32'), + ('erf', paddle.erf, erf_ag, + (np.random.rand(100, 200), ), None, 'float32'), + ('gelu', paddle.nn.functional.gelu, lambda xs: gelu_ag(xs[0]), + (np.random.rand(10, 20, 30), ), None, 'float32'), + ('gelu_approximate', + lambda x: paddle.nn.functional.gelu(x, approximate=True), + lambda xs: gelu_ag(xs[0], approximate=True), + (np.random.rand(10, 20, 30), ), None, 'float32'), + ('sigmoid', paddle.nn.functional.sigmoid, sigmoid_ag, + (np.random.rand(10, 20), ), None, 'float32'), + )) class TestGradWithHigherOrder(unittest.TestCase): def setUp(self): diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index ff0ae86a59f090..502c10783e887d 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -504,15 +504,8 @@ def rsqrt(x, out=None): return _simple_unop(LayerHelper('rsqrt_p', **locals())) -@REGISTER_FN('uniform_random_p', 'ShapeTensor', 'ShapeTensorList', 'Out') -def uniform_random(dtype, - min_value, - max_value, - seed, - shape=None, - shape_t=None, - shape_tl=None, - out=None): +@REGISTER_FN('uniform_random_p', 'Out') +def uniform_random(dtype, min_value, max_value, seed, shape=None, out=None): attrs = { 'shape': shape, 'dtype': dtype, @@ -523,18 +516,5 @@ def uniform_random(dtype, helper = LayerHelper('uniform_random_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype) - if shape_tl: - helper.append_op(type=helper.layer_type, - inputs={'ShapeTensorList': shape_tl}, - outputs={'Out': out}, - attrs=attrs) - elif shape_t: - helper.append_op(type=helper.layer_type, - inputs={'ShapeTensor': shape_t}, - outputs={'Out': out}, - attrs=attrs) - else: - helper.append_op(type=helper.layer_type, - outputs={'Out': out}, - attrs=attrs) + helper.append_op(type=helper.layer_type, outputs={'Out': out}, attrs=attrs) return out diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 46267b0a9f46ff..8a2f94145cd06f 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -29,8 +29,6 @@ lookup_orig2prim, lookup_prim2orig, lookup_transpose, op_position_inputs, op_position_output) from .utils import INT_DTYPE_2_STRING, get_output_var_list -from paddle.fluid.data_feeder import convert_dtype -from paddle.fluid.framework import convert_np_dtype_to_dtype_ def _orig2prim(op, *args): @@ -213,8 +211,7 @@ def fill_any_like_orig2prim(op, x): return fill_const(value=op.attr('value'), shape=x.shape, dtype=x.dtype) return fill_const(value=op.attr('value'), shape=x.shape, - dtype=convert_np_dtype_to_dtype_( - convert_dtype(INT_DTYPE_2_STRING[op.attr('dtype')]))) + dtype=paddle.dtype(op.attr('dtype'))) @REGISTER_ORIG2PRIM('fill_constant') @@ -328,6 +325,13 @@ def slice_orig2prim(op, ends_t, ends_tl, x, starts_t, starts_tl): return y +@REGISTER_ORIG2PRIM('sigmoid') +def sigmoid_orig2prim(op, x): + return div( + fill_const(value=1.0, shape=x.shape, dtype=x.dtype), + (add(fill_const(value=1.0, shape=x.shape, dtype=x.dtype), exp(neg(x))))) + + @REGISTER_ORIG2PRIM('p_norm') def p_norm_orig2prim(op, x): @@ -467,34 +471,16 @@ def dropout_orig2prim(op, seed_t, x): @REGISTER_ORIG2PRIM('uniform_random') def uniform_random_orig2prim(op, shape_t, shape_tl): + if shape_t or shape_tl: + raise TypeError( + 'uniform_random_orig2prim currently not support ShapeTensor input or ShapeTensorList input.' + ) min_value = op.attr('min') max_value = op.attr('max') seed = op.attr('seed') - dtype = convert_np_dtype_to_dtype_( - convert_dtype(INT_DTYPE_2_STRING[op.attr('dtype')])) - if shape_tl: - return uniform_random(dtype, - min_value, - max_value, - seed, - shape=None, - shape_t=None, - shape_tl=shape_tl) - elif shape_t: - return uniform_random(dtype, - min_value, - max_value, - seed, - shape=None, - shape_t=shape_t, - shape_tl=None) - return uniform_random(dtype, - min_value, - max_value, - seed, - shape=op.attr('shape'), - shape_t=None, - shape_tl=None) + dtype = paddle.dtype(op.attr('dtype')) + shape = op.attr('shape') + return uniform_random(dtype, min_value, max_value, seed, shape=shape) @REGISTER_ORIG2PRIM('reduce_sum') @@ -701,19 +687,7 @@ def bernoulli_prim2orig(op): @REGISTER_PRIM2ORIG('uniform_random_p') -def uniform_random_prim2orig(op, shape_t, shape_tl): - if shape_tl: - return paddle.uniform(shape=shape_tl, - dtype=INT_DTYPE_2_STRING[op.attr('dtype')], - min=op.attr('min'), - max=op.attr('max'), - seed=op.attr('seed')) - elif shape_t: - return paddle.uniform(shape=shape_t, - dtype=INT_DTYPE_2_STRING[op.attr('dtype')], - min=op.attr('min'), - max=op.attr('max'), - seed=op.attr('seed')) +def uniform_random_prim2orig(op): return paddle.uniform(shape=op.attr('shape'), dtype=INT_DTYPE_2_STRING[op.attr('dtype')], min=op.attr('min'), @@ -1114,16 +1088,6 @@ def rsqrt_jvp(op, x_dot): return y_dot -#TODO(wanghao) : uniform_random_p shouldn't have jvp rules. -#But for compatibility, we return None in uniform_random_jvp. -#We want users not to compute meaningless derivatives. -#In the future we will add pruning logic to be compatible -#with operators which has no derivatives defined but with inputs. -@REGISTER_JVP('uniform_random_p') -def uniform_random_jvp(op, shape_t_dot, shape_tl_dot): - return None - - ## Register transpose rules