Merge pull request #4415 from xdelaruelle/unit_tests_modules4

Run unit tests on an updated versions of Modules: v4.5.3 + v5.3.1

.github/workflows/unit_tests.yml

@@ -14,7 +14,8 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
         lmod8: Lmod-8.7.6
-        modules4: modules-4.1.4
+        modules4: modules-4.5.3
+        modules5: modules-5.3.1
     steps:
       - run: "true"
   build:
@@ -28,6 +29,7 @@ jobs:
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
           - ${{needs.setup.outputs.lmod8}}
           - ${{needs.setup.outputs.modules4}}
+          - ${{needs.setup.outputs.modules5}}
         lc_all: [""]
         include:
           # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
@@ -158,7 +160,7 @@ jobs:
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod

test/framework/easyblock.py

@@ -284,8 +284,11 @@ def test_make_module_extend_modpath(self):
         txt = eb.make_module_extend_modpath()
         if module_syntax == 'Tcl':
             regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
-            home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } '
-            home += r'else { concat "HOME_NOT_DEFINED" } \]'
+            if self.modtool.supports_tcl_getenv:
+                home = r'\[getenv HOME "HOME_NOT_DEFINED"\]'
+            else:
+                home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } '
+                home += r'else { concat "HOME_NOT_DEFINED" } \]'
             fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
             regexs.extend([
                 # extension for user modules is guarded
@@ -327,9 +330,12 @@ def test_make_module_extend_modpath(self):
         for envvar in list_of_envvars:
             if module_syntax == 'Tcl':
                 regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
-                module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar
-                module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar
-                module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED')
+                if self.modtool.supports_tcl_getenv:
+                    module_envvar = r'\[getenv %s "%s"]' % (envvar, envvar + '_NOT_DEFINED')
+                else:
+                    module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar
+                    module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar
+                    module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED')
                 fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
                 regexs.extend([
                     # extension for user modules is guarded
@@ -754,21 +760,26 @@ def test_make_module_dep(self):
             eb.prepare_step()
 
         if get_module_syntax() == 'Tcl':
-            tc_load = '\n'.join([
-                "if { ![ is-loaded gompi/2018a ] } {",
-                "    module load gompi/2018a",
-                "}",
-            ])
-            fftw_load = '\n'.join([
-                "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
-                "    module load FFTW/3.3.7-gompi-2018a",
-                "}",
-            ])
-            lapack_load = '\n'.join([
-                "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {",
-                "    module load OpenBLAS/0.2.20-GCC-6.4.0-2.28",
-                "}",
-            ])
+            if self.modtool.supports_safe_auto_load:
+                tc_load = "module load gompi/2018a"
+                fftw_load = "module load FFTW/3.3.7-gompi-2018a"
+                lapack_load = "module load OpenBLAS/0.2.20-GCC-6.4.0-2.28"
+            else:
+                tc_load = '\n'.join([
+                    "if { ![ is-loaded gompi/2018a ] } {",
+                    "    module load gompi/2018a",
+                    "}",
+                ])
+                fftw_load = '\n'.join([
+                    "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
+                    "    module load FFTW/3.3.7-gompi-2018a",
+                    "}",
+                ])
+                lapack_load = '\n'.join([
+                    "if { ![ is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28 ] } {",
+                    "    module load OpenBLAS/0.2.20-GCC-6.4.0-2.28",
+                    "}",
+                ])
         elif get_module_syntax() == 'Lua':
             tc_load = '\n'.join([
                 'if not ( isloaded("gompi/2018a") ) then',
@@ -798,12 +809,18 @@ def test_make_module_dep(self):
         }
 
         if get_module_syntax() == 'Tcl':
-            fftw_load = '\n'.join([
-                "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
-                "    module unload FFTW",
-                "    module load FFTW/3.3.7-gompi-2018a",
-                "}",
-            ])
+            if self.modtool.supports_safe_auto_load:
+                fftw_load = '\n'.join([
+                    "module unload FFTW",
+                    "module load FFTW/3.3.7-gompi-2018a",
+                ])
+            else:
+                fftw_load = '\n'.join([
+                    "if { ![ is-loaded FFTW/3.3.7-gompi-2018a ] } {",
+                    "    module unload FFTW",
+                    "    module load FFTW/3.3.7-gompi-2018a",
+                    "}",
+                ])
         elif get_module_syntax() == 'Lua':
             fftw_load = '\n'.join([
                 'if not ( isloaded("FFTW/3.3.7-gompi-2018a") ) then',

test/framework/easyconfig.py

@@ -4792,7 +4792,10 @@ def test_recursive_module_unload(self):
             recursive_unload_pat = r'if mode\(\) == "unload" or not \( isloaded\("%(mod)s"\) \) then\n'
             recursive_unload_pat += r'\s*load\("%(mod)s"\)'
         else:
-            guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s'
+            if self.modtool.supports_safe_auto_load:
+                guarded_load_pat = r'\nmodule load %(mod)s'
+            else:
+                guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s'
             recursive_unload_pat = r'if { \[ module-info mode remove \] \|\| \!\[ is-loaded %(mod)s \] } {\n'
             recursive_unload_pat += r'\s*module load %(mod)s'
 
@@ -4831,10 +4834,16 @@ def test_recursive_module_unload(self):
             eb_bis.prepare_step()
             eb_bis.make_module_step()
         modtxt = read_file(test_module)
-        fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
-        self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
-        fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
-        self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
+        if self.modtool.supports_safe_auto_load:
+            fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
+            self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
+            fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+            self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg)
+        else:
+            fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
+            self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
+            fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+            self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
 
         # recursive_mod_unload build option is honored
         update_build_option('recursive_mod_unload', True)
@@ -4844,10 +4853,16 @@ def test_recursive_module_unload(self):
             eb.prepare_step()
             eb.make_module_step()
         modtxt = read_file(test_module)
-        fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
-        self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
-        fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
-        self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
+        if self.modtool.supports_safe_auto_load:
+            fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt)
+            self.assertTrue(guarded_load_regex.search(modtxt), fail_msg)
+            fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+            self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg)
+        else:
+            fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt)
+            self.assertFalse(guarded_load_regex.search(modtxt), fail_msg)
+            fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt)
+            self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg)
 
         # disabling via easyconfig parameter works even when recursive_mod_unload build option is enabled
         self.assertTrue(build_option('recursive_mod_unload'))

test/framework/toolchain.py

@@ -50,6 +50,7 @@
 from easybuild.tools.environment import setvar
 from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir
 from easybuild.tools.filetools import read_file, symlink, write_file, which
+from easybuild.tools.modules import EnvironmentModules
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.toolchain.mpi import get_mpi_cmd_template
@@ -470,6 +471,10 @@ def test_cray_reset(self):
         init_config(build_options={'optarch': 'test', 'silent': True})
 
         for tcname in ['CrayGNU', 'CrayCCE', 'CrayIntel']:
+            # Cray* modules do not unload other Cray* modules thus loading a second Cray* module
+            # makes environment inconsistent which is not allowed by Environment Modules tool
+            if isinstance(self.modtool, EnvironmentModules):
+                self.modtool.purge()
             tc = self.get_toolchain(tcname, version='2015.06-XC')
             tc.set_options({'dynamic': True})
             with self.mocked_stdout_stderr():
@@ -2208,6 +2213,10 @@ def test_independence(self):
         # purposely obtain toolchains several times in a row, value for $CFLAGS should not change
         for _ in range(3):
             for tcname, tcversion in toolchains:
+                # Cray* modules do not unload other Cray* modules thus loading a second Cray* module
+                # makes environment inconsistent which is not allowed by Environment Modules tool
+                if isinstance(self.modtool, EnvironmentModules):
+                    self.modtool.purge()
                 tc = get_toolchain({'name': tcname, 'version': tcversion}, {},
                                    mns=ActiveMNS(), modtool=self.modtool)
                 # also check whether correct compiler flag for OpenMP is used while we're at it

test/framework/toy_build.py

@@ -2330,6 +2330,7 @@ def test_reproducibility_ext_easyblocks(self):
 
     def test_toy_toy(self):
         """Test building two easyconfigs in a single go, with one depending on the other."""
+
         topdir = os.path.dirname(os.path.abspath(__file__))
         toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
         toy_ec_txt = read_file(toy_ec_file)
@@ -2341,9 +2342,17 @@ def test_toy_toy(self):
         ])
         write_file(ec1, ec1_txt)
 
+        # adapt toy easyconfig for toy2 to produce a modulefile with a dedicated
+        # name ('toy2' instead of 'toy')
         ec2 = os.path.join(self.test_prefix, 'toy2.eb')
         ec2_txt = '\n'.join([
             toy_ec_txt,
+            "name = 'toy2'",
+            "easyblock = 'EB_toy'",
+            "sources = ['toy/toy-0.0.tar.gz']",
+            "patches = []",
+            "sanity_check_paths = { 'files': ['bin/toy2'], 'dirs': ['bin']}",
+            "prebuildopts = 'mv toy.source toy2.c &&'",
             "versionsuffix = '-two'",
             "dependencies = [('toy', '0.0', '-one')]",
         ])
@@ -2353,7 +2362,7 @@ def test_toy_toy(self):
             self._test_toy_build(ec_file=self.test_prefix, verify=False)
 
         mod1 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-one')
-        mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-two')
+        mod2 = os.path.join(self.test_installpath, 'modules', 'all', 'toy2', '0.0-two')
         if get_module_syntax() == 'Lua':
             mod1 += '.lua'
             mod2 += '.lua'
@@ -2366,15 +2375,15 @@ def test_toy_toy(self):
         self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt))
 
         # Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load
-        reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod')
-        dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env')
+        reprod_dir = os.path.join(self.test_installpath, 'software', 'toy2', '0.0-two', 'easybuild', 'reprod')
+        dumpenv_script = os.path.join(reprod_dir, 'toy2-0.0-two.env')
         reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script)
         self.assertExists(reprod_dumpenv)
 
         # Check contents of the dumpenv script
         patterns = [
             """#!/bin/bash""",
-            """# usage: source toy-0.0-two.env""",
+            """# usage: source toy2-0.0-two.env""",
             # defining build env
             """module load toy/0.0-one""",
             """# (no build environment defined)""",
@@ -3200,11 +3209,22 @@ def test_toy_multi_deps(self):
                 'end',
             ])
         else:
-            expected = '\n'.join([
-                'if { ![ is-loaded GCC/4.6.3 ] && ![ is-loaded GCC/7.3.0-2.30 ] } {',
-                '    module load GCC/4.6.3',
-                '}',
-            ])
+            if not self.modtool.supports_safe_auto_load:
+                expected = '\n'.join([
+                    'if { ![ is-loaded GCC/4.6.3 ] && ![ is-loaded GCC/7.3.0-2.30 ] } {',
+                    '    module load GCC/4.6.3',
+                    '}',
+                ])
+            else:
+                expected = '\n'.join([
+                    '',
+                    "if { [ module-info mode remove ] || [ is-loaded GCC/7.3.0-2.30 ] } {",
+                    "    module load GCC",
+                    '} else {',
+                    "    module load GCC/4.6.3",
+                    '}',
+                    '',
+                ])
 
         self.assertIn(expected, toy_mod_txt)