From 3f4bcec832f0f3e3c2f2886ccb777be58fe528af Mon Sep 17 00:00:00 2001 From: jonatanm Date: Tue, 24 Sep 2019 17:22:56 -0700 Subject: [PATCH 1/2] Multiple channel support in Gluon PReLU --- python/mxnet/gluon/nn/activations.py | 11 ++++++++--- tests/python/unittest/test_gluon.py | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/python/mxnet/gluon/nn/activations.py b/python/mxnet/gluon/nn/activations.py index a3baae004311..4402469fe2d1 100644 --- a/python/mxnet/gluon/nn/activations.py +++ b/python/mxnet/gluon/nn/activations.py @@ -120,7 +120,10 @@ class PReLU(HybridBlock): ---------- alpha_initializer : Initializer Initializer for the `embeddings` matrix. - + in_channels : int + Number of channels (alpha parameters) to learn. Can either be 1 + or `n` where `n` is the size of the second dimension of the input + tensor. Inputs: - **data**: input tensor with arbitrary shape. @@ -128,10 +131,12 @@ class PReLU(HybridBlock): Outputs: - **out**: output tensor with the same shape as `data`. """ - def __init__(self, alpha_initializer=initializer.Constant(0.25), **kwargs): + def __init__(self, alpha_initializer=initializer.Constant(0.25), + in_channels=1, **kwargs): super(PReLU, self).__init__(**kwargs) with self.name_scope(): - self.alpha = self.params.get('alpha', shape=(1,), init=alpha_initializer) + self.alpha = self.params.get('alpha', shape=(in_channels,), + init=alpha_initializer) def hybrid_forward(self, F, x, alpha): return F.LeakyReLU(x, gamma=alpha, act_type='prelu', name='fwd') diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index d11443ca839a..2fdb7393660a 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -1240,6 +1240,11 @@ def selu(x): x = point_to_validate.reshape((1, 3, 2)) assert_almost_equal(prelu(x).asnumpy(), mx.nd.where(x >= 0, x, 0.25 * x).asnumpy()) + multichannel_init = mx.initializer.Constant(mx.nd.array([0.1, 0.25, 0.5])) + prelu_multichannel = mx.gluon.nn.PReLU(alpha_initializer=multichannel_init, in_channels=3) + prelu_multichannel.initialize() + assert_almost_equal(prelu_multichannel(x).asnumpy(), np.array([[-0.01, 0.1], [-0.025, 0.1], [-0.05, 0.1]])) + gelu = mx.gluon.nn.GELU() def gelu_test(x): CUBE_CONSTANT = 0.044715 From 015ad7743874f9dbfb54b1d5d4267e94c5864186 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 11 Dec 2019 15:00:19 -0800 Subject: [PATCH 2/2] Update activations.py --- python/mxnet/gluon/nn/activations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/gluon/nn/activations.py b/python/mxnet/gluon/nn/activations.py index 4402469fe2d1..1b9ce91dd2aa 100644 --- a/python/mxnet/gluon/nn/activations.py +++ b/python/mxnet/gluon/nn/activations.py @@ -120,7 +120,7 @@ class PReLU(HybridBlock): ---------- alpha_initializer : Initializer Initializer for the `embeddings` matrix. - in_channels : int + in_channels : int, default 1 Number of channels (alpha parameters) to learn. Can either be 1 or `n` where `n` is the size of the second dimension of the input tensor.