Merge pull request #3907 from boegel/ext_extract_cmd

pick up custom extract_cmd specified for extension

easybuild/framework/easyblock.py

@@ -572,8 +572,12 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
 
                         if fetch_files:
                             src = self.fetch_source(source, checksums, extension=True)
-                            # copy 'path' entry to 'src' for use with extensions
-                            ext_src.update({'src': src['path']})
+                            ext_src.update({
+                                # keep track of custom extract command (if any)
+                                'extract_cmd': src['cmd'],
+                                # copy 'path' entry to 'src' for use with extensions
+                                'src': src['path'],
+                            })
 
                     else:
                         # use default template for name of source file if none is specified

easybuild/framework/extension.py

@@ -119,6 +119,7 @@ def __init__(self, mself, ext, extra_params=None):
 
         # list of source/patch files: we use an empty list as default value like in EasyBlock
         self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values)
+        self.src_extract_cmd = self.ext.get('extract_cmd', None)
         self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values)
         self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values)
 

easybuild/framework/extensioneasyblock.py

@@ -118,7 +118,7 @@ def run(self, unpack_src=False):
         if unpack_src:
             targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name))
             self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options,
-                                        change_into_dir=False)
+                                        change_into_dir=False, cmd=self.src_extract_cmd)
 
             # setting start dir must be done from unpacked source directory for extension,
             # because start_dir value is usually a relative path (if it is set)

easybuild/tools/filetools.py

@@ -456,11 +456,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced
     _log.debug("Unpacking %s in directory %s", fn, abs_dest)
     cwd = change_dir(abs_dest)
 
-    if not cmd:
-        cmd = extract_cmd(fn, overwrite=overwrite)
-    else:
+    if cmd:
         # complete command template with filename
         cmd = cmd % fn
+        _log.debug("Using specified command to unpack %s: %s", fn, cmd)
+    else:
+        cmd = extract_cmd(fn, overwrite=overwrite)
+        _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd)
+
     if not cmd:
         raise EasyBuildError("Can't extract file %s with unknown filetype", fn)
 
@@ -1366,7 +1369,7 @@ def find_extension(filename):
     if res:
         ext = res.group('ext')
     else:
-        raise EasyBuildError('Unknown file type for file %s', filename)
+        raise EasyBuildError("%s has unknown file extension", filename)
 
     return ext
 
@@ -1379,7 +1382,9 @@ def extract_cmd(filepath, overwrite=False):
     ext = find_extension(filename)
     target = filename[:-len(ext)]
 
+    # find_extension will either return an extension listed in EXTRACT_CMDS, or raise an error
     cmd_tmpl = EXTRACT_CMDS[ext.lower()]
+
     if overwrite:
         if 'unzip -qq' in cmd_tmpl:
             cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o')

test/framework/filetools.py

@@ -114,6 +114,9 @@ def test_extract_cmd(self):
 
         self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True))
 
+        error_pattern = "test.foo has unknown file extension"
+        self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_cmd, 'test.foo')
+
     def test_find_extension(self):
         """Test find_extension function."""
         tests = [

test/framework/toy_build.py

@@ -1362,6 +1362,31 @@ def test_toy_extension_sources(self):
             write_file(test_ec, test_ec_txt)
             self.test_toy_build(ec_file=test_ec, raise_error=True)
 
+    def test_toy_extension_extract_cmd(self):
+        """Test for custom extract_cmd specified for an extension."""
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec)
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = '\n'.join([
+            toy_ec_txt,
+            'exts_list = [',
+            '   ("bar", "0.0", {',
+            # deliberately incorrect custom extract command, just to verify that it's picked up
+            '       "sources": [{',
+            '           "filename": "bar-%(version)s.tar.gz",',
+            '           "extract_cmd": "unzip %s",',
+            '       }],',
+            '   }),',
+            ']',
+        ])
+        write_file(test_ec, test_ec_txt)
+
+        error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]"
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
+                              raise_error=True, verbose=False)
+
     def test_toy_extension_sources_git_config(self):
         """Test install toy that includes extensions with 'sources' spec including 'git_config'."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')