From 6961a2090f026c80eaa13c4537442a1b7444c819 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 28 Jun 2018 20:25:02 +0000 Subject: [PATCH 01/20] Fix build issue with CUDNN=0 --- Jenkinsfile | 39 ++++++++++++++++++++++++++++++++++ ci/docker/runtime_functions.sh | 18 ++++++++++++++++ src/operator/nn/convolution.cu | 12 +++++------ 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7156dc163919..34cd67ab0449 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -281,6 +281,17 @@ try { } } }, + 'GPU: MKLDNN_CUDNNOFF': { + node('mxnetlinux-cpu') { + ws('workspace/build-mkldnn-gpu-nocudnn') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_cuda', false) + pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + } + } + } + }, 'GPU: CUDA9.1+cuDNN7': { node('mxnetlinux-cpu') { ws('workspace/build-gpu') { @@ -645,6 +656,34 @@ try { } } }, + 'Python2: MKLDNN-GPU-NOCUDNN': { + node('mxnetlinux-gpu') { + ws('workspace/ut-python2-mkldnn-gpu-nocudnn') { + try { + init_git() + unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + python2_gpu_ut('ubuntu_gpu') + publish_test_coverage() + } finally { + collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu_nocudnn.xml') + } + } + } + }, + 'Python3: MKLDNN-GPU-NOCUDNN': { + node('mxnetlinux-gpu') { + ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { + try { + init_git() + unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + python3_gpu_ut('ubuntu_gpu') + publish_test_coverage() + } finally { + collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') + } + } + } + }, 'Python3: CentOS 7 CPU': { node('mxnetlinux-cpu') { ws('workspace/build-centos7-cpu') { diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e49639903f92..52bcb46a3a00 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -457,6 +457,24 @@ build_ubuntu_gpu_mkldnn() { report_ccache_usage } +build_ubuntu_gpu_mkldnn_nocudnn() { + set -ex + + build_ccache_wrappers + + make \ + DEV=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + USE_MKLDNN=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=0 \ + -j$(nproc) + + report_ccache_usage +} + build_ubuntu_gpu_cuda91_cudnn7() { set -ex # unfortunately this build has problems in 3rdparty dependencies with ccache and make diff --git a/src/operator/nn/convolution.cu b/src/operator/nn/convolution.cu index 9f61212d5c78..797557e35df9 100644 --- a/src/operator/nn/convolution.cu +++ b/src/operator/nn/convolution.cu @@ -90,9 +90,9 @@ void ConvolutionCompute(const nnvm::NodeAttrs& attrs, int dtype = inputs[conv::kData].type_flag_; #if CUDNN_MAJOR < 5 - if (param_.layout.value() != kNCW && - param_.layout.value() != kNCHW && - param_.layout.value() != kNCDHW) { + if (param.layout.value() != kNCW && + param.layout.value() != kNCHW && + param.layout.value() != kNCDHW) { // Need CuDNN > 5.0 for layout support. use MXNet implementation MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { ConvolutionOp op; @@ -168,9 +168,9 @@ void ConvolutionGradCompute(const nnvm::NodeAttrs& attrs, int dtype = out_grad.type_flag_; #if CUDNN_MAJOR < 5 - if (param_.layout.value() != kNCW && - param_.layout.value() != kNCHW && - param_.layout.value() != kNCDHW) { + if (param.layout.value() != kNCW && + param.layout.value() != kNCHW && + param.layout.value() != kNCDHW) { // Need CuDNN > 5.0 for layout support. use MXNet implementation MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { ConvolutionOp op; From 7bdf6c7b1921a67728fea2968940b6fd73af29f6 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 28 Jun 2018 20:47:25 +0000 Subject: [PATCH 02/20] Fix nocudnn func name --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 34cd67ab0449..29b56e38d907 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -286,7 +286,7 @@ try { ws('workspace/build-mkldnn-gpu-nocudnn') { timeout(time: max_time, unit: 'MINUTES') { init_git() - docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_cuda', false) + docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) } } From 06815b4a21fe885b7d6d8c81d79bae3b6c921eff Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 28 Jun 2018 20:49:21 +0000 Subject: [PATCH 03/20] Remove python2 tests --- Jenkinsfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 29b56e38d907..10f3394addce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -656,20 +656,6 @@ try { } } }, - 'Python2: MKLDNN-GPU-NOCUDNN': { - node('mxnetlinux-gpu') { - ws('workspace/ut-python2-mkldnn-gpu-nocudnn') { - try { - init_git() - unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - python2_gpu_ut('ubuntu_gpu') - publish_test_coverage() - } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu_nocudnn.xml') - } - } - } - }, 'Python3: MKLDNN-GPU-NOCUDNN': { node('mxnetlinux-gpu') { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { From def0842b38eebca90c1358cb808b6400fb7d61b1 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 28 Jun 2018 21:12:46 +0000 Subject: [PATCH 04/20] Remove CPP package test --- ci/docker/runtime_functions.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 52bcb46a3a00..87b0068d2436 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -464,7 +464,6 @@ build_ubuntu_gpu_mkldnn_nocudnn() { make \ DEV=1 \ - USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_MKLDNN=1 \ USE_CUDA=1 \ From e7e60848bf4d2df1c315e209f5465da1517f371f Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Mon, 2 Jul 2018 18:51:37 +0000 Subject: [PATCH 05/20] Check assert raises when cudnn disabled for op tests on gpu --- include/mxnet/c_api.h | 7 +++++++ python/mxnet/test_utils.py | 11 +++++++++++ src/c_api/c_api.cc | 9 +++++++++ tests/python/gpu/test_operator_gpu.py | 5 +++-- tests/python/unittest/common.py | 15 ++++++++++++++- tests/python/unittest/test_gluon_rnn.py | 13 ++++++++++++- 6 files changed, 56 insertions(+), 4 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 4dd858a51c4b..8f17484a5521 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -2317,6 +2317,13 @@ MXNET_DLL int MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, MXNET_DLL int MXNDArrayCreateFromSharedMem(int shared_pid, int shared_id, const mx_uint *shape, mx_uint ndim, int dtype, NDArrayHandle *out); +/*! + * \brief Check if cudnn is enabled for MXNet + * \param curr returns the current status. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXCUDNNIsEnabled(bool* curr); + #ifdef __cplusplus } diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 19fe07495987..64af2b83d210 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -30,6 +30,7 @@ import logging import bz2 import zipfile +import ctypes from contextlib import contextmanager import numpy as np import numpy.testing as npt @@ -48,6 +49,8 @@ from .ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID from .ndarray import array from .symbol import Symbol +from .base import _LIB, check_call + def default_context(): @@ -68,6 +71,14 @@ def default_dtype(): return np.float32 +def cudnn_enabled(): + """Check if cudnn enabled for the MXNet installation + """ + curr = ctypes.c_bool() + check_call(_LIB.MXCUDNNIsEnabled(ctypes.byref(curr))) + return curr.value + + def get_atol(atol=None): """Get default numerical threshold for regression test.""" # _TODO: get from env variable, different threshold might diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 467118b9921e..6fd6be7bbd22 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -1285,6 +1285,15 @@ int MXRtcCudaKernelCall(CudaKernelHandle handle, int dev_id, void** args, API_END(); } +int MXCUDNNIsEnabled(bool *curr) { + API_BEGIN(); +#if MXNET_USE_CUDNN + *curr = true; +#else + *curr = false; +#endif + API_END(); +} int MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, int* shared_id) { API_BEGIN(); diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index e4b1917f398d..ef993cee0418 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -25,14 +25,14 @@ import numpy as np import unittest from nose.tools import assert_raises -from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal +from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal, cudnn_enabled from mxnet.base import MXNetError from mxnet import autograd from numpy.testing import assert_allclose curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.insert(0, os.path.join(curr_path, '../unittest')) -from common import setup_module, with_seed, teardown +from common import setup_module, with_seed, teardown, assert_raises_cudnn_disabled from test_operator import * from test_optimizer import * from test_random import * @@ -1614,6 +1614,7 @@ def check_rnn_layer_w_rand_inputs(layer): assert_almost_equal(g.asnumpy(), c.asnumpy(), rtol=1e-2, atol=1e-6) @with_seed() +@assert_raises_cudnn_disabled() def test_rnn_layer(): check_rnn_layer(gluon.rnn.RNN(100, num_layers=3)) check_rnn_layer(gluon.rnn.RNN(100, activation='tanh', num_layers=3)) diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index b38c851984d2..0a652f97525e 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -20,13 +20,14 @@ import numpy as np import random import shutil +from mxnet.base import MXNetError curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.append(os.path.join(curr_path, '../common/')) sys.path.insert(0, os.path.join(curr_path, '../../../python')) import models from contextlib import contextmanager -from nose.tools import make_decorator +from nose.tools import make_decorator, assert_raises import tempfile def assertRaises(expected_exception, func, *args, **kwargs): @@ -94,6 +95,18 @@ def random_seed(seed=None): random.seed(next_seed) +def assert_raises_cudnn_disabled(): + def test_helper(orig_test): + @make_decorator(orig_test) + def test_new(*args, **kwargs): + if mx.test_utils.cudnn_enabled() or mx.context.current_context().device_type == 'cpu': + orig_test(*args, **kwargs) + else: + assert_raises((MXNetError, RuntimeError), orig_test, *args, **kwargs) + return test_new + return test_helper + + def with_seed(seed=None): """ A decorator for nosetests test functions that manages rng seeds. diff --git a/tests/python/unittest/test_gluon_rnn.py b/tests/python/unittest/test_gluon_rnn.py index 9dbcb3b3be8d..2225f1a8e543 100644 --- a/tests/python/unittest/test_gluon_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -21,8 +21,9 @@ from numpy.testing import assert_allclose import unittest from mxnet.test_utils import almost_equal +from common import assert_raises_cudnn_disabled - +@assert_raises_cudnn_disabled() def test_rnn(): cell = gluon.rnn.RNNCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] @@ -47,6 +48,7 @@ def test_lstm(): assert outs == [(10, 100), (10, 100), (10, 100)] +@assert_raises_cudnn_disabled() def test_lstm_forget_bias(): forget_bias = 2.0 stack = gluon.rnn.SequentialRNNCell() @@ -67,6 +69,8 @@ def test_lstm_forget_bias(): forget_bias * np.ones(100, ), np.zeros((2 * 100,))]) assert_allclose(mod.get_params()[0][bias_argument].asnumpy(), expected_bias) + +@assert_raises_cudnn_disabled() def test_lstm_cpu_inference(): # should behave the same as lstm cell EXPECTED_LSTM_OUTPUT = np.array([[[0.72045636, 0.72045636, 0.95215213, 0.95215213], @@ -82,6 +86,7 @@ def test_lstm_cpu_inference(): rtol=1e-3, atol=1e-5) +@assert_raises_cudnn_disabled() def test_gru(): cell = gluon.rnn.GRUCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] @@ -149,6 +154,7 @@ def test_residual_bidirectional(): assert np.array_equal(outputs[1].asnumpy(), expected_outputs) +@assert_raises_cudnn_disabled() def test_stack(): cell = gluon.rnn.SequentialRNNCell() for i in range(5): @@ -171,6 +177,7 @@ def test_stack(): assert outs == [(10, 100), (10, 100), (10, 100)] +@assert_raises_cudnn_disabled() def test_bidirectional(): cell = gluon.rnn.BidirectionalCell( gluon.rnn.LSTMCell(100, prefix='rnn_l0_'), @@ -221,6 +228,7 @@ def check_rnn_forward(layer, inputs, deterministic=True): mx.test_utils.assert_almost_equal(np_dx, inputs.grad.asnumpy(), rtol=1e-3, atol=1e-5) +@assert_raises_cudnn_disabled() def test_rnn_cells(): check_rnn_forward(gluon.rnn.LSTMCell(100, input_size=200), mx.nd.ones((8, 3, 200))) check_rnn_forward(gluon.rnn.RNNCell(100, input_size=200), mx.nd.ones((8, 3, 200))) @@ -275,6 +283,7 @@ def check_rnn_layer_forward(layer, inputs, states=None, run_only=False): mx.test_utils.assert_almost_equal(np_dx, inputs.grad.asnumpy(), rtol=1e-3, atol=1e-5) +@assert_raises_cudnn_disabled() def test_rnn_layers(): check_rnn_layer_forward(gluon.rnn.RNN(10, 2), mx.nd.ones((8, 3, 20))) check_rnn_layer_forward(gluon.rnn.RNN(10, 2, bidirectional=True), mx.nd.ones((8, 3, 20)), mx.nd.ones((4, 3, 10))) @@ -387,6 +396,8 @@ def test_cell_fill_shape(): check_rnn_forward(cell, mx.nd.ones((2, 3, 7))) assert cell.i2h_weight.shape[1] == 7, cell.i2h_weight.shape[1] + +@assert_raises_cudnn_disabled() def test_layer_fill_shape(): layer = gluon.rnn.LSTM(10) layer.hybridize() From 06316829f9eb2fcfe5edb7daad45275338c9389d Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Mon, 2 Jul 2018 19:04:04 +0000 Subject: [PATCH 06/20] Add line --- tests/python/unittest/test_gluon_rnn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unittest/test_gluon_rnn.py b/tests/python/unittest/test_gluon_rnn.py index 879d23c1478b..15a246ca8aec 100644 --- a/tests/python/unittest/test_gluon_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -23,6 +23,7 @@ from mxnet.test_utils import almost_equal from common import assert_raises_cudnn_disabled + @assert_raises_cudnn_disabled() def test_rnn(): cell = gluon.rnn.RNNCell(100, prefix='rnn_') From 2d11ade2d0b49e28cf9e6b18949b16eb6ff474b3 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Mon, 2 Jul 2018 20:33:16 +0000 Subject: [PATCH 07/20] Remove whitespace --- tests/python/unittest/test_operator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 461fb63514c1..8a93f1f34262 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -157,17 +157,17 @@ def test_rnntanh_bidirectional(): fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='rnn_tanh', bidirectional=True, get_next_state=True, prefix='') - + stack = mx.rnn.SequentialRNNCell() stack.add(mx.rnn.BidirectionalCell( mx.rnn.RNNCell(H, activation='tanh', prefix='l0_'), mx.rnn.RNNCell(H, activation='tanh', prefix='r0_'), - output_prefix='bi_rnntanh_0_')) + output_prefix='bi_rnntanh_0_')) stack.add(mx.rnn.BidirectionalCell( mx.rnn.RNNCell(H, activation='tanh', prefix='l1_'), mx.rnn.RNNCell(H, activation='tanh', prefix='r1_'), output_prefix='bi_rnntanh_1_')) - + check_rnn_consistency(fused, stack, T, N, I, H, 'write') check_rnn_consistency(fused, stack, T, N, I, H, 'add') check_rnn_consistency(fused, stack, T, N, I, H, 'null') @@ -194,12 +194,12 @@ def test_rnnrelu_bidirectional(): fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='rnn_relu', bidirectional=True, get_next_state=True, prefix='') - + stack = mx.rnn.SequentialRNNCell() stack.add(mx.rnn.BidirectionalCell( mx.rnn.RNNCell(H, activation='relu', prefix='l0_'), mx.rnn.RNNCell(H, activation='relu', prefix='r0_'), - output_prefix='bi_rnnrelu_0_')) + output_prefix='bi_rnnrelu_0_')) stack.add(mx.rnn.BidirectionalCell( mx.rnn.RNNCell(H, activation='relu', prefix='l1_'), mx.rnn.RNNCell(H, activation='relu', prefix='r1_'), @@ -3073,7 +3073,7 @@ def l2norm(input_data, axis=0, keepdims=True): atol=1e-2 if dtype is np.float16 else 1e-5, ctx=ctx) # check gradient check_numeric_gradient(norm_sym, [in_data], numeric_eps=epsilon, rtol=1e-2, atol=1e-3) - + def test_layer_norm(): for dtype, forward_check_eps in zip([np.float16, np.float32, np.float64], From f338866b6c832d6123e08cf2dc308efc4e6d60f2 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Mon, 2 Jul 2018 22:08:24 +0000 Subject: [PATCH 08/20] add decorator for other ops --- tests/python/gpu/test_operator_gpu.py | 7 +++++++ tests/python/unittest/test_gluon.py | 3 ++- tests/python/unittest/test_operator.py | 7 ++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index a56196cfb5fc..2c805c31b580 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -422,6 +422,7 @@ def test_3d_batchnorm(fix_gamma, use_global_stats): @with_seed(1234) +@assert_raises_cudnn_disabled() def test_convolution_with_type(): sym1 = mx.sym.Convolution(num_filter=3, kernel=(3,3), name='conv') @@ -1316,6 +1317,7 @@ def check_rnn_consistency(cell1, cell2): assert_allclose(mod1.get_outputs()[0].asnumpy(), mod2.get_outputs()[0].asnumpy(), rtol=1e-2, atol=1e-4) @with_seed() +@assert_raises_cudnn_disabled() def test_rnn(): fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='rnn_relu', prefix='') @@ -1327,6 +1329,7 @@ def test_rnn(): check_rnn_consistency(stack, fused) @with_seed() +@assert_raises_cudnn_disabled() def test_lstm_forget_bias(): forget_bias = 2.0 fused = mx.rnn.FusedRNNCell(10, forget_bias=forget_bias, num_layers=2, mode='lstm', prefix='') @@ -1348,6 +1351,7 @@ def test_lstm_forget_bias(): assert_allclose(args[bias_name].asnumpy(), expected_bias) @with_seed() +@assert_raises_cudnn_disabled() def test_gru(): fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='') @@ -1359,6 +1363,7 @@ def test_gru(): check_rnn_consistency(stack, fused) @with_seed() +@assert_raises_cudnn_disabled() def test_bidirectional(): fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='', bidirectional=True) @@ -1377,6 +1382,7 @@ def test_bidirectional(): check_rnn_consistency(stack, fused) @with_seed() +@assert_raises_cudnn_disabled() def test_unfuse(): for mode in ['rnn_tanh', 'rnn_relu', 'lstm', 'gru']: fused = mx.rnn.FusedRNNCell( @@ -1558,6 +1564,7 @@ def test_deformable_convolution_options(): name='deformable_conv') @with_seed() +@assert_raises_cudnn_disabled() def test_residual_fused(): cell = mx.rnn.ResidualCell( mx.rnn.FusedRNNCell(50, num_layers=3, mode='lstm', diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index 43777bb8032b..d752dbaa3038 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -20,7 +20,7 @@ from mxnet.gluon import nn from mxnet.test_utils import assert_almost_equal from mxnet.ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID -from common import setup_module, with_seed, assertRaises, teardown +from common import setup_module, with_seed, assertRaises, teardown, assert_raises_cudnn_disabled import numpy as np from numpy.testing import assert_array_equal from nose.tools import raises, assert_raises @@ -1258,6 +1258,7 @@ def record_name(block): @with_seed() +@assert_raises_cudnn_disabled() def test_summary(): net = gluon.model_zoo.vision.resnet50_v1() net.initialize() diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 8a93f1f34262..aba73bba0d23 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -25,7 +25,7 @@ from numpy.testing import assert_allclose, assert_array_equal from mxnet.test_utils import * from mxnet.base import py_str, MXNetError -from common import setup_module, with_seed, teardown +from common import setup_module, with_seed, teardown, assert_raises_cudnn_disabled import unittest def check_rnn_consistency(cell1, cell2, T, N, I, H, grad_req): @@ -103,6 +103,7 @@ def test_lstm_bidirectional(): check_rnn_consistency(fused, stack, T, N, I, H, 'null') @with_seed() +@assert_raises_cudnn_disabled() def test_gru_sym(): T, N, I, H = 5, 32, 800, 800 fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='gru', get_next_state=True, prefix='') @@ -116,6 +117,7 @@ def test_gru_sym(): check_rnn_consistency(fused, stack, T, N, I, H, 'null') @with_seed() +@assert_raises_cudnn_disabled() def test_gru_bidirectional(): T, N, I, H = 5, 20, 800, 800 @@ -138,6 +140,7 @@ def test_gru_bidirectional(): check_rnn_consistency(fused, stack, T, N, I, H, 'null') @with_seed() +@assert_raises_cudnn_disabled() def test_rnntanh_sym(): T, N, I, H = 5, 32, 800, 800 @@ -152,6 +155,7 @@ def test_rnntanh_sym(): check_rnn_consistency(fused, stack, T, N, I, H, 'null') @with_seed() +@assert_raises_cudnn_disabled() def test_rnntanh_bidirectional(): T, N, I, H = 5, 20, 800, 800 @@ -173,6 +177,7 @@ def test_rnntanh_bidirectional(): check_rnn_consistency(fused, stack, T, N, I, H, 'null') @with_seed() +@assert_raises_cudnn_disabled() def test_rnnrelu_sym(): T, N, I, H = 5, 32, 200, 200 From a5c2c79ea1a75235c54561fac2fa12a4ce5e23b8 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 5 Jul 2018 20:15:22 +0000 Subject: [PATCH 09/20] Add and remove assert --- tests/python/unittest/test_gluon_rnn.py | 3 --- tests/python/unittest/test_operator.py | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/python/unittest/test_gluon_rnn.py b/tests/python/unittest/test_gluon_rnn.py index 073eeb8f6ead..6167f660d2c1 100644 --- a/tests/python/unittest/test_gluon_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -212,7 +212,6 @@ def test_residual_bidirectional(): assert np.array_equal(outputs[1].asnumpy(), expected_outputs) -@assert_raises_cudnn_disabled() def test_stack(): cell = gluon.rnn.SequentialRNNCell() for i in range(5): @@ -235,7 +234,6 @@ def test_stack(): assert outs == [(10, 100), (10, 100), (10, 100)] -@assert_raises_cudnn_disabled() def test_hybridstack(): cell = gluon.rnn.HybridSequentialRNNCell() for i in range(5): @@ -366,7 +364,6 @@ def check_rnn_forward(layer, inputs, deterministic=True): mx.test_utils.assert_almost_equal(np_dx, input_grads, rtol=1e-3, atol=1e-5) -@assert_raises_cudnn_disabled() def test_rnn_cells(): check_rnn_forward(gluon.rnn.LSTMCell(100, input_size=200), mx.nd.ones((8, 3, 200))) check_rnn_forward(gluon.rnn.RNNCell(100, input_size=200), mx.nd.ones((8, 3, 200))) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 832de9621784..2fd1f198e86b 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -71,6 +71,7 @@ def check_rnn_consistency(cell1, cell2, T, N, I, H, grad_req): @with_seed() +@assert_raises_cudnn_disabled() def test_lstm_sym(): T, N, I, H = 5, 32, 800, 800 fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='lstm', get_next_state=True, prefix='') @@ -84,6 +85,7 @@ def test_lstm_sym(): check_rnn_consistency(fused, stack, T, N, I, H, 'null') @with_seed() +@assert_raises_cudnn_disabled() def test_lstm_bidirectional(): T, N, I, H = 5, 20, 800, 800 fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='lstm', From 71d7d724d0f310849980633587d498453a26111c Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 5 Jul 2018 21:42:42 +0000 Subject: [PATCH 10/20] Fix op and common --- tests/python/unittest/common.py | 8 ++++++-- tests/python/unittest/test_operator.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index 0a652f97525e..e5608f1619fb 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -95,14 +95,18 @@ def random_seed(seed=None): random.seed(next_seed) -def assert_raises_cudnn_disabled(): +def assert_raises_cudnn_disabled(assertion_error=False): def test_helper(orig_test): @make_decorator(orig_test) def test_new(*args, **kwargs): if mx.test_utils.cudnn_enabled() or mx.context.current_context().device_type == 'cpu': orig_test(*args, **kwargs) else: - assert_raises((MXNetError, RuntimeError), orig_test, *args, **kwargs) + if assertion_error: + errors = (MXNetError, RuntimeError, AssertionError) + else: + errors = (MXNetError, RuntimeError) + assert_raises(errors, orig_test, *args, **kwargs) return test_new return test_helper diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 2fd1f198e86b..2669a8fd5565 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -2391,6 +2391,7 @@ def test_flip(): @with_seed() +@assert_raises_cudnn_disabled(assertion_error=True) def test_stn(): np.set_printoptions(threshold=np.nan) num_filter = 2 # conv of loc net From 65d01e0f4849228a38a814adc84671d143c7ea2d Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Thu, 5 Jul 2018 22:42:29 +0000 Subject: [PATCH 11/20] Fix merge issue --- tests/python/unittest/test_operator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 2669a8fd5565..e5099d5847e3 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -25,7 +25,7 @@ import itertools from numpy.testing import assert_allclose, assert_array_equal from mxnet.test_utils import * -from mxnet.base import py_str, MXNetError +from mxnet.base import py_str, MXNetError, _as_list from common import setup_module, with_seed, teardown, assert_raises_cudnn_disabled import unittest From 4ce776cb146c59559553171b5301df79f04415d0 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Tue, 10 Jul 2018 01:13:28 +0000 Subject: [PATCH 12/20] Remove C API --- Jenkinsfile | 22 +++++++++++++--------- python/mxnet/test_utils.py | 8 -------- src/c_api/c_api.cc | 10 ---------- tests/python/gpu/test_operator_gpu.py | 2 +- tests/python/unittest/common.py | 3 ++- 5 files changed, 16 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 94ffa1bbb9c1..f0691e59f278 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -295,11 +295,13 @@ try { 'GPU: MKLDNN_CUDNNOFF': { node('mxnetlinux-cpu') { ws('workspace/build-mkldnn-gpu-nocudnn') { - timeout(time: max_time, unit: 'MINUTES') { + withEnv(['CUDNN_DISABLED=ON'] { + timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - } + } + } } } }, @@ -681,13 +683,15 @@ try { 'Python3: MKLDNN-GPU-NOCUDNN': { node('mxnetlinux-gpu') { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { - try { - init_git() - unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - python3_gpu_ut('ubuntu_gpu') - publish_test_coverage() - } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') + withEnv(['CUDNN_DISABLED=ON'] { + try { + init_git() + unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + python3_gpu_ut('ubuntu_gpu') + publish_test_coverage() + } finally { + collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') + } } } } diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 2fae9ef1b8ef..d2675f10e000 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -71,14 +71,6 @@ def default_dtype(): return np.float32 -def cudnn_enabled(): - """Check if cudnn enabled for the MXNet installation - """ - curr = ctypes.c_bool() - check_call(_LIB.MXCUDNNIsEnabled(ctypes.byref(curr))) - return curr.value - - def get_atol(atol=None): """Get default numerical threshold for regression test.""" # _TODO: get from env variable, different threshold might diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 6fd6be7bbd22..c746f6861bca 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -1285,16 +1285,6 @@ int MXRtcCudaKernelCall(CudaKernelHandle handle, int dev_id, void** args, API_END(); } -int MXCUDNNIsEnabled(bool *curr) { - API_BEGIN(); -#if MXNET_USE_CUDNN - *curr = true; -#else - *curr = false; -#endif - API_END(); -} - int MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, int* shared_id) { API_BEGIN(); NDArray* arr = reinterpret_cast(handle); diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index d1a4af99521c..162e2474d918 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -25,7 +25,7 @@ import numpy as np import unittest from nose.tools import assert_raises -from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal, cudnn_enabled +from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal from mxnet.base import MXNetError from mxnet import autograd from numpy.testing import assert_allclose diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index e5608f1619fb..e9929fe301a8 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -99,7 +99,8 @@ def assert_raises_cudnn_disabled(assertion_error=False): def test_helper(orig_test): @make_decorator(orig_test) def test_new(*args, **kwargs): - if mx.test_utils.cudnn_enabled() or mx.context.current_context().device_type == 'cpu': + cudnn_disabled = (os.getenv('CUDNN_DISABLED') == "ON") + if not cudnn_disabled or mx.context.current_context().device_type == 'cpu': orig_test(*args, **kwargs) else: if assertion_error: From 556dee179ed9640d1411024d57ccdcbf376f2fac Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Tue, 10 Jul 2018 06:50:35 +0000 Subject: [PATCH 13/20] Fix --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f0691e59f278..c67e5e6ec7de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -295,7 +295,7 @@ try { 'GPU: MKLDNN_CUDNNOFF': { node('mxnetlinux-cpu') { ws('workspace/build-mkldnn-gpu-nocudnn') { - withEnv(['CUDNN_DISABLED=ON'] { + withEnv(['CUDNN_DISABLED=ON']) { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) @@ -683,9 +683,8 @@ try { 'Python3: MKLDNN-GPU-NOCUDNN': { node('mxnetlinux-gpu') { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { - withEnv(['CUDNN_DISABLED=ON'] { + withEnv(['CUDNN_DISABLED=ON']) { try { - init_git() unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) python3_gpu_ut('ubuntu_gpu') publish_test_coverage() From 67d00a35185b47e88e616ba23880fe8a5a6f15d5 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Tue, 10 Jul 2018 06:55:41 +0000 Subject: [PATCH 14/20] Fix lint --- python/mxnet/test_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index d2675f10e000..e963d158446d 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -30,7 +30,6 @@ import logging import bz2 import zipfile -import ctypes from contextlib import contextmanager import numpy as np import numpy.testing as npt @@ -49,8 +48,6 @@ from .ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID from .ndarray import array from .symbol import Symbol -from .base import _LIB, check_call - def default_context(): From 266ba862180cd9ea82147b5eff59dfe71fa98ae2 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Tue, 10 Jul 2018 18:25:24 +0000 Subject: [PATCH 15/20] Add init git --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index c67e5e6ec7de..229c8364676e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -685,6 +685,7 @@ try { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { withEnv(['CUDNN_DISABLED=ON']) { try { + init_git() unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) python3_gpu_ut('ubuntu_gpu') publish_test_coverage() From e837777ce36aad412bba7bf410775ba004637e78 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Tue, 10 Jul 2018 19:12:06 +0000 Subject: [PATCH 16/20] Rename CUDNN_DISABLED env variable --- Jenkinsfile | 4 ++-- tests/python/unittest/common.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 229c8364676e..edd9e7a5f61b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -295,7 +295,7 @@ try { 'GPU: MKLDNN_CUDNNOFF': { node('mxnetlinux-cpu') { ws('workspace/build-mkldnn-gpu-nocudnn') { - withEnv(['CUDNN_DISABLED=ON']) { + withEnv(['CUDNN_OFF_TEST_ONLY=ON']) { timeout(time: max_time, unit: 'MINUTES') { init_git() docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) @@ -683,7 +683,7 @@ try { 'Python3: MKLDNN-GPU-NOCUDNN': { node('mxnetlinux-gpu') { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { - withEnv(['CUDNN_DISABLED=ON']) { + withEnv(['CUDNN_OFF_TEST_ONLY=ON']) { try { init_git() unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index e9929fe301a8..04631a6be6a9 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -99,7 +99,7 @@ def assert_raises_cudnn_disabled(assertion_error=False): def test_helper(orig_test): @make_decorator(orig_test) def test_new(*args, **kwargs): - cudnn_disabled = (os.getenv('CUDNN_DISABLED') == "ON") + cudnn_disabled = (os.getenv('CUDNN_OFF_TEST_ONLY') == "ON") if not cudnn_disabled or mx.context.current_context().device_type == 'cpu': orig_test(*args, **kwargs) else: From 2f590eec2b80d0fe229ddde65ed4ea2c3fc40886 Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Tue, 10 Jul 2018 23:32:26 +0000 Subject: [PATCH 17/20] Add a runtime function for nocudnn --- Jenkinsfile | 33 ++++++++++++++++++--------------- ci/docker/runtime_functions.sh | 8 ++++++++ tests/python/unittest/common.py | 2 +- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index edd9e7a5f61b..9ddaeaeb4bc1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -163,6 +163,13 @@ def python3_gpu_ut(docker_container_name) { } } +// Python 3 NOCUDNN +def python3_gpu_ut_nocudnn(docker_container_name) { + timeout(time: max_time, unit: 'MINUTES') { + docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_nocudnn', true) + } +} + try { stage('Sanity Check') { parallel 'Lint': { @@ -295,12 +302,10 @@ try { 'GPU: MKLDNN_CUDNNOFF': { node('mxnetlinux-cpu') { ws('workspace/build-mkldnn-gpu-nocudnn') { - withEnv(['CUDNN_OFF_TEST_ONLY=ON']) { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) - pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - } + timeout(time: max_time, unit: 'MINUTES') { + init_git() + docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) + pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) } } } @@ -683,15 +688,13 @@ try { 'Python3: MKLDNN-GPU-NOCUDNN': { node('mxnetlinux-gpu') { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { - withEnv(['CUDNN_OFF_TEST_ONLY=ON']) { - try { - init_git() - unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - python3_gpu_ut('ubuntu_gpu') - publish_test_coverage() - } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') - } + try { + init_git() + unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + python3_gpu_ut_nocudnn('ubuntu_gpu') + publish_test_coverage() + } finally { + collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') } } } diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 3f0a850b06fc..01d2ce03f480 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -628,6 +628,14 @@ unittest_ubuntu_python3_gpu() { nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } +unittest_ubuntu_python3_gpu_nocudnn() { + set -ex + export PYTHONPATH=./python/ + export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export CUDNN_OFF_TEST_ONLY=true + nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu +} + # quantization gpu currently only runs on P3 instances # need to separte it from unittest_ubuntu_python2_gpu() unittest_ubuntu_python2_quantization_gpu() { diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index 04631a6be6a9..65c1886bf1ea 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -99,7 +99,7 @@ def assert_raises_cudnn_disabled(assertion_error=False): def test_helper(orig_test): @make_decorator(orig_test) def test_new(*args, **kwargs): - cudnn_disabled = (os.getenv('CUDNN_OFF_TEST_ONLY') == "ON") + cudnn_disabled = (os.getenv('CUDNN_OFF_TEST_ONLY') == "true") if not cudnn_disabled or mx.context.current_context().device_type == 'cpu': orig_test(*args, **kwargs) else: From 749981def9112adac131ec4ceea5b1e20ecf0e8f Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Wed, 11 Jul 2018 21:09:38 +0000 Subject: [PATCH 18/20] Remove MXCudnnIsenabled --- include/mxnet/c_api.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 12c587689167..75147cfd706d 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -2371,13 +2371,6 @@ MXNET_DLL int MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, MXNET_DLL int MXNDArrayCreateFromSharedMem(int shared_pid, int shared_id, const mx_uint *shape, mx_uint ndim, int dtype, NDArrayHandle *out); -/*! - * \brief Check if cudnn is enabled for MXNet - * \param curr returns the current status. - * \return 0 when success, -1 when failure happens - */ -MXNET_DLL int MXCUDNNIsEnabled(bool* curr); - #ifdef __cplusplus } From 9573913e800ff12013699301a31e328d21bc792e Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Wed, 11 Jul 2018 21:14:47 +0000 Subject: [PATCH 19/20] Add comment for disabled test --- tests/python/unittest/test_operator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index e941063bc38a..7638a7e69e62 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -2391,6 +2391,7 @@ def test_flip(): @with_seed() +# The test is disabled with USE_CUDA=ON and USE_CUDNN=OFF because of failures with the SpatialTransformer op. Tracked in #11568 @assert_raises_cudnn_disabled(assertion_error=True) def test_stn(): np.set_printoptions(threshold=np.nan) From e8d87dd32d080eb1e89583c5df8f3ffd805fe22a Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Wed, 11 Jul 2018 21:21:32 +0000 Subject: [PATCH 20/20] Add full link in comment --- tests/python/unittest/test_operator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 7638a7e69e62..b38489c146d5 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -2391,7 +2391,8 @@ def test_flip(): @with_seed() -# The test is disabled with USE_CUDA=ON and USE_CUDNN=OFF because of failures with the SpatialTransformer op. Tracked in #11568 +# The test is disabled with USE_CUDA=ON and USE_CUDNN=OFF because of failures with the SpatialTransformer op. +# Tracked at https://github.com/apache/incubator-mxnet/issues/11568 @assert_raises_cudnn_disabled(assertion_error=True) def test_stn(): np.set_printoptions(threshold=np.nan)