Merge pull request #3667 from boegel/install_extensions

add initial/experimental support for installing extensions in parallel

easybuild/framework/extension.py

@@ -40,7 +40,7 @@
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
-from easybuild.tools.run import run_cmd
+from easybuild.tools.run import check_async_cmd, run_cmd
 from easybuild.tools.py2vs3 import string_type
 
 
@@ -139,6 +139,12 @@ def __init__(self, mself, ext, extra_params=None):
                                key, name, version, value)
 
         self.sanity_check_fail_msgs = []
+        self.async_cmd_info = None
+        self.async_cmd_output = None
+        self.async_cmd_check_cnt = None
+        # initial read size should be relatively small,
+        # to avoid hanging for a long time until desired output is available in async_cmd_check
+        self.async_cmd_read_size = 1024
 
     @property
     def name(self):
@@ -160,18 +166,67 @@ def prerun(self):
         """
         pass
 
-    def run(self):
+    def run(self, *args, **kwargs):
         """
-        Actual installation of a extension.
+        Actual installation of an extension.
         """
         pass
 
+    def run_async(self, *args, **kwargs):
+        """
+        Asynchronous installation of an extension.
+        """
+        raise NotImplementedError
+
     def postrun(self):
         """
         Stuff to do after installing a extension.
         """
         self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', []))
 
+    def async_cmd_start(self, cmd, inp=None):
+        """
+        Start installation asynchronously using specified command.
+        """
+        self.async_cmd_output = ''
+        self.async_cmd_check_cnt = 0
+        self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True)
+
+    def async_cmd_check(self):
+        """
+        Check progress of installation command that was started asynchronously.
+
+        :return: True if command completed, False otherwise
+        """
+        if self.async_cmd_info is None:
+            raise EasyBuildError("No installation command running asynchronously for %s", self.name)
+        elif self.async_cmd_info is False:
+            self.log.info("No asynchronous command was started for extension %s", self.name)
+            return True
+        else:
+            self.log.debug("Checking on installation of extension %s...", self.name)
+            # use small read size, to avoid waiting for a long time until sufficient output is produced
+            res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size)
+            self.async_cmd_output += res['output']
+            if res['done']:
+                self.log.info("Installation of extension %s completed!", self.name)
+                self.async_cmd_info = None
+            else:
+                self.async_cmd_check_cnt += 1
+                self.log.debug("Installation of extension %s still running (checked %d times)",
+                               self.name, self.async_cmd_check_cnt)
+                # increase read size after sufficient checks,
+                # to avoid that installation hangs due to output buffer filling up...
+                if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2):
+                    self.async_cmd_read_size *= 2
+
+            return res['done']
+
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name)
+
     @property
     def toolchain(self):
         """

easybuild/tools/config.py

@@ -270,6 +270,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'module_extensions',
         'module_only',
         'package',
+        'parallel_extensions_install',
         'read_only_installdir',
         'remove_ghost_install_dirs',
         'rebuild',

easybuild/tools/options.py

@@ -456,6 +456,8 @@ def override_options(self):
                              'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
             'parallel': ("Specify (maximum) level of parallellism used during build procedure",
                          'int', 'store', None),
+            'parallel-extensions-install': ("Install list of extensions in parallel (if supported)",
+                                            None, 'store_true', False),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
                                       None, 'store_true', True),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
@@ -890,6 +892,10 @@ def postprocess(self):
         # set tmpdir
         self.tmpdir = set_tmpdir(self.options.tmpdir)
 
+        # early check for opt-in to installing extensions in parallel (experimental feature)
+        if self.options.parallel_extensions_install:
+            self.log.experimental("installing extensions in parallel")
+
         # take --include options into account (unless instructed otherwise)
         if self.with_include:
             self._postprocess_include()

easybuild/tools/output.py

@@ -242,7 +242,7 @@ def extensions_progress_bar():
     Get progress bar to show progress for installing extensions.
     """
     progress_bar = Progress(
-        TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"),
+        TextColumn("[bold blue]{task.description}"),
         BarColumn(),
         TimeElapsedColumn(),
     )

test/framework/run.py

@@ -611,11 +611,15 @@ def test_run_cmd_async(self):
         # check asynchronous running of failing command
         error_test_cmd = "echo 'FAIL!' >&2; exit 123"
         cmd_info = run_cmd(error_test_cmd, asynchronous=True)
+        time.sleep(1)
         error_pattern = 'cmd ".*" exited with exit code 123'
         self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info)
 
         cmd_info = run_cmd(error_test_cmd, asynchronous=True)
         res = check_async_cmd(*cmd_info, fail_on_error=False)
+        # keep checking until command is fully done
+        while not res['done']:
+            res = check_async_cmd(*cmd_info, fail_on_error=False)
         self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"})
 
         # also test with a command that produces a lot of output,

test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py

@@ -30,7 +30,8 @@
 
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
-from easybuild.easyblocks.toy import EB_toy
+from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd
+from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.run import run_cmd
 
 
@@ -45,20 +46,59 @@ def extra_options():
         }
         return ExtensionEasyBlock.extra_options(extra_vars=extra_vars)
 
-    def run(self):
-        """Build toy extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        deps = {
+            'bar': [],
+            'barbar': ['bar'],
+            'ls': [],
+        }
+        if self.name in deps:
+            return deps[self.name]
+        else:
+            raise EasyBuildError("Dependencies for %s are unknown!", self.name)
+
+    def run(self, *args, **kwargs):
+        """
+        Install toy extension.
+        """
         if self.src:
-            super(Toy_Extension, self).run(unpack_src=True)
-            EB_toy.configure_step(self.master, name=self.name)
             EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts'])
 
             if self.cfg['toy_ext_param']:
                 run_cmd(self.cfg['toy_ext_param'])
 
-            EB_toy.install_step(self.master, name=self.name)
-
             return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name)
 
+    def prerun(self):
+        """
+        Prepare installation of toy extension.
+        """
+        super(Toy_Extension, self).prerun()
+
+        if self.src:
+            super(Toy_Extension, self).run(unpack_src=True)
+            EB_toy.configure_step(self.master, name=self.name)
+
+    def run_async(self):
+        """
+        Install toy extension asynchronously.
+        """
+        if self.src:
+            cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
+            self.async_cmd_start(cmd)
+        else:
+            self.async_cmd_info = False
+
+    def postrun(self):
+        """
+        Wrap up installation of toy extension.
+        """
+        super(Toy_Extension, self).postrun()
+
+        EB_toy.install_step(self.master, name=self.name)
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for toy extensions."""
         self.log.info("Loaded modules: %s", self.modules_tool.list())

test/framework/sandbox/easybuild/easyblocks/t/toy.py

@@ -41,6 +41,19 @@
 from easybuild.tools.run import run_cmd
 
 
+def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts):
+    """
+    Compose command to build toy.
+    """
+
+    cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % {
+        'name': name,
+        'prebuildopts': prebuildopts,
+        'buildopts': buildopts,
+    }
+    return cmd
+
+
 class EB_toy(ExtensionEasyBlock):
     """Support for building/installing toy."""
 
@@ -92,17 +105,13 @@ def configure_step(self, name=None):
 
     def build_step(self, name=None, buildopts=None):
         """Build toy."""
-
         if buildopts is None:
             buildopts = self.cfg['buildopts']
-
         if name is None:
             name = self.name
-        run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % {
-            'name': name,
-            'prebuildopts': self.cfg['prebuildopts'],
-            'buildopts': buildopts,
-        })
+
+        cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts)
+        run_cmd(cmd)
 
     def install_step(self, name=None):
         """Install toy."""
@@ -118,11 +127,38 @@ def install_step(self, name=None):
         mkdir(libdir, parents=True)
         write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper())
 
-    def run(self):
-        """Install toy as extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+        if self.name == 'toy':
+            return ['bar', 'barbar']
+        else:
+            raise EasyBuildError("Dependencies for %s are unknown!", self.name)
+
+    def prerun(self):
+        """
+        Prepare installation of toy as extension.
+        """
         super(EB_toy, self).run(unpack_src=True)
         self.configure_step()
+
+    def run(self):
+        """
+        Install toy as extension.
+        """
         self.build_step()
+
+    def run_async(self):
+        """
+        Asynchronous installation of toy as extension.
+        """
+        cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
+        self.async_cmd_start(cmd)
+
+    def postrun(self):
+        """
+        Wrap up installation of toy as extension.
+        """
         self.install_step()
 
     def make_module_step(self, fake=False):

test/framework/toy_build.py

@@ -1777,6 +1777,45 @@ def test_module_only_extensions(self):
         self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True)
         self.assertTrue(os.path.exists(toy_mod))
 
+    def test_toy_exts_parallel(self):
+        """
+        Test parallel installation of extensions (--parallel-extensions-install)
+        """
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+        toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod += '.lua'
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += '\n' + '\n'.join([
+            "exts_list = [",
+            "    ('ls'),",
+            "    ('bar', '0.0'),",
+            "    ('barbar', '0.0', {",
+            "        'start_dir': 'src',",
+            "    }),",
+            "    ('toy', '0.0'),",
+            "]",
+            "sanity_check_commands = ['barbar', 'toy']",
+            "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}",
+        ])
+        write_file(test_ec, test_ec_txt)
+
+        args = ['--parallel-extensions-install', '--experimental', '--force']
+        stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        self.assertEqual(stderr, '')
+        expected_stdout = '\n'.join([
+            "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)",
+            "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)",
+            "== 3 out of 4 extensions installed (0 queued, 1 running: toy)",
+            "== 4 out of 4 extensions installed (0 queued, 0 running: )",
+            '',
+        ])
+        self.assertEqual(stdout, expected_stdout)
+
     def test_backup_modules(self):
         """Test use of backing up of modules with --module-only."""