From 47cf140a35765e8d44474c552294819e107a57e1 Mon Sep 17 00:00:00 2001 From: Ingmar Schoegl Date: Mon, 10 May 2021 05:29:19 -0500 Subject: [PATCH 1/5] [CI] Run pytest with verbose output --- test/python/runCythonTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/runCythonTests.py b/test/python/runCythonTests.py index 3105ba3241..e9ae85ee9f 100644 --- a/test/python/runCythonTests.py +++ b/test/python/runCythonTests.py @@ -84,7 +84,7 @@ def addError(self, test, err): if not subsets: subsets.append(str(base)) - pytest_args = ["-raP", "--durations=50", "--junitxml=pytest.xml"] + pytest_args = ["-v", "-raP", "--durations=50", "--junitxml=pytest.xml"] if fast_fail: pytest_args.insert(0, "-x") From 2f7d3f1ebceedcb5f54091d94cee51213329738b Mon Sep 17 00:00:00 2001 From: Ingmar Schoegl Date: Mon, 10 May 2021 07:36:46 -0500 Subject: [PATCH 2/5] [CI] Hide longest test durations by default --- SConstruct | 11 +++++++---- test/SConscript | 3 +++ test/python/runCythonTests.py | 17 +++++++++++------ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/SConstruct b/SConstruct index dc4d29e174..214bf66f06 100644 --- a/SConstruct +++ b/SConstruct @@ -390,7 +390,7 @@ config_options = [ To install to the current user's 'site-packages' directory, use 'python_prefix=USER'.""", defaults.python_prefix, PathVariable.PathAccept), - EnumVariable( + EnumVariable( 'matlab_toolbox', """This variable controls whether the MATLAB toolbox will be built. If set to 'y', you will also need to set the value of the 'matlab_path' @@ -663,14 +663,17 @@ config_options = [ BoolVariable( "fast_fail_tests", """If enabled, tests will exit at the first failure.""", - False, - ), + False), BoolVariable( "skip_slow_tests", """If enabled, skip a subset of tests that are known to have long runtimes. Skipping these may be desirable when running with options that cause tests to run slowly, like disabling optimization or activating code profiling.""", - False) + False), + BoolVariable( + "show_long_tests", + """If enabled, duration of slowest tests will be shown.""", + False), ] opts.AddVariables(*config_options) diff --git a/test/SConscript b/test/SConscript index 0a93b5901a..638addc31d 100644 --- a/test/SConscript +++ b/test/SConscript @@ -3,6 +3,7 @@ import subprocess from xml.etree import ElementTree from os.path import join as pjoin import time +import sys from buildutils import * @@ -141,6 +142,8 @@ def addPythonTest(testname, subdir, script, interpreter, outfile, cmdargs = args.split() if env["fast_fail_tests"]: cmdargs.insert(0, "fast_fail") + if env["show_long_tests"]: + cmdargs.insert(0, "show_long") if env["skip_slow_tests"]: environ["CT_SKIP_SLOW"] = "1" diff --git a/test/python/runCythonTests.py b/test/python/runCythonTests.py index e9ae85ee9f..c11f0c5487 100644 --- a/test/python/runCythonTests.py +++ b/test/python/runCythonTests.py @@ -68,12 +68,15 @@ def addError(self, test, err): print('* INFO: Git commit:', cantera.__git_commit__, '\n') sys.stdout.flush() - if len(sys.argv) > 1 and sys.argv[1] == "fast_fail": + subset_start = 1 + fast_fail = False + show_long = False + if "fast_fail" in sys.argv: fast_fail = True - subset_start = 2 - else: - fast_fail = False - subset_start = 1 + subset_start += 1 + if "show_long" in sys.argv: + show_long = True + subset_start += 1 if pytest is not None: base = Path(cantera.__file__).parent.joinpath('test') @@ -84,7 +87,9 @@ def addError(self, test, err): if not subsets: subsets.append(str(base)) - pytest_args = ["-v", "-raP", "--durations=50", "--junitxml=pytest.xml"] + pytest_args = ["-v", "-raP", "--junitxml=pytest.xml"] + if show_long: + pytest_args += ["--durations=50"] if fast_fail: pytest_args.insert(0, "-x") From 825c077b3795e48e755f4dc6e5ef7373efd7d034 Mon Sep 17 00:00:00 2001 From: Ingmar Schoegl Date: Wed, 9 Jun 2021 09:04:54 -0500 Subject: [PATCH 3/5] [SCons] Add verbose_tests flag --- SConstruct | 4 ++++ test/SConscript | 2 ++ test/python/runCythonTests.py | 8 +++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 214bf66f06..c33af0aa8d 100644 --- a/SConstruct +++ b/SConstruct @@ -674,6 +674,10 @@ config_options = [ "show_long_tests", """If enabled, duration of slowest tests will be shown.""", False), + BoolVariable( + "verbose_tests", + """If enabled, verbose test output will be shown.""", + False), ] opts.AddVariables(*config_options) diff --git a/test/SConscript b/test/SConscript index 638addc31d..a4979b6223 100644 --- a/test/SConscript +++ b/test/SConscript @@ -144,6 +144,8 @@ def addPythonTest(testname, subdir, script, interpreter, outfile, cmdargs.insert(0, "fast_fail") if env["show_long_tests"]: cmdargs.insert(0, "show_long") + if env["verbose_tests"]: + cmdargs.insert(0, "verbose") if env["skip_slow_tests"]: environ["CT_SKIP_SLOW"] = "1" diff --git a/test/python/runCythonTests.py b/test/python/runCythonTests.py index c11f0c5487..3946f34c65 100644 --- a/test/python/runCythonTests.py +++ b/test/python/runCythonTests.py @@ -71,12 +71,16 @@ def addError(self, test, err): subset_start = 1 fast_fail = False show_long = False + verbose = False if "fast_fail" in sys.argv: fast_fail = True subset_start += 1 if "show_long" in sys.argv: show_long = True subset_start += 1 + if "verbose" in sys.argv: + verbose = True + subset_start += 1 if pytest is not None: base = Path(cantera.__file__).parent.joinpath('test') @@ -87,11 +91,13 @@ def addError(self, test, err): if not subsets: subsets.append(str(base)) - pytest_args = ["-v", "-raP", "--junitxml=pytest.xml"] + pytest_args = ["-raP", "--junitxml=pytest.xml"] if show_long: pytest_args += ["--durations=50"] if fast_fail: pytest_args.insert(0, "-x") + if verbose: + pytest_args.insert(0, "-v") ret_code = pytest.main(pytest_args + subsets) sys.exit(ret_code) From b8a6923001cba3b62bc44d4c632d1f5dc7938481 Mon Sep 17 00:00:00 2001 From: Ingmar Schoegl Date: Wed, 9 Jun 2021 13:15:30 -0500 Subject: [PATCH 4/5] [SCons] Do not show logging WARNINGS unless verbose_tests=yes --- test/python/runCythonTests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/python/runCythonTests.py b/test/python/runCythonTests.py index 3946f34c65..0f69179a66 100644 --- a/test/python/runCythonTests.py +++ b/test/python/runCythonTests.py @@ -98,6 +98,8 @@ def addError(self, test, err): pytest_args.insert(0, "-x") if verbose: pytest_args.insert(0, "-v") + else: + pytest_args.append("--log-level=ERROR") ret_code = pytest.main(pytest_args + subsets) sys.exit(ret_code) From 03c1f3370fef8fc3781373ddaeced70ea2e87c19 Mon Sep 17 00:00:00 2001 From: Ingmar Schoegl Date: Mon, 14 Jun 2021 12:41:11 -0500 Subject: [PATCH 5/5] Show detailed testing info on GH Actions --- .github/workflows/main.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a287396d5..1021aac514 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,8 @@ jobs: - name: Build Cantera run: python3 `which scons` build env_vars=all -j2 debug=n --debug=time - name: Test Cantera - run: python3 `which scons` test --debug=time + run: + python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time env: GITHUB_ACTIONS: "true" @@ -83,7 +84,8 @@ jobs: - name: Build Cantera run: python3 `which scons` build env_vars=all -j3 debug=n --debug=time - name: Test Cantera - run: python3 `which scons` test --debug=time + run: + python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time env: GITHUB_ACTIONS: "true" @@ -117,7 +119,8 @@ jobs: python3 `which scons` build blas_lapack_libs=lapack,blas coverage=y \ optimize=n skip_slow_tests=y no_optimize_flags=-DNDEBUG env_vars=all -j2 --debug=time - name: Test Cantera - run: python3 `which scons` test --debug=time + run: + python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time env: GITHUB_ACTIONS: "true" - name: Upload Coverage to Codecov @@ -246,7 +249,7 @@ jobs: extra_lib_dirs=$CONDA_PREFIX/lib system_fmt=y system_eigen=y system_yamlcpp=y \ system_sundials=y blas_lapack_libs='lapack,blas' -j2 VERBOSE=True debug=n - name: Test Cantera - run: scons test + run: scons test show_long_tests=yes verbose_tests=yes cython-latest: name: Test pre-release version of Cython @@ -272,7 +275,7 @@ jobs: - name: Build Cantera run: python3 `which scons` build blas_lapack_libs=lapack,blas python_package='full' -j2 debug=n - name: Test Cantera - run: python3 `which scons` test + run: python3 `which scons` test show_long_tests=yes verbose_tests=yes windows: name: ${{ matrix.os }}, MSVC ${{ matrix.vs-toolset }}, Python ${{ matrix.python-version }} @@ -338,6 +341,6 @@ jobs: msvc_version=${{ matrix.vs-toolset }} f90_interface=n --debug=time shell: cmd - name: Test Cantera - run: scons test --debug=time + run: scons test show_long_tests=yes verbose_tests=yes --debug=time env: GITHUB_ACTIONS: "true"