diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 29c1495f40..a0a941e02f 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -572,8 +572,12 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: src = self.fetch_source(source, checksums, extension=True) - # copy 'path' entry to 'src' for use with extensions - ext_src.update({'src': src['path']}) + ext_src.update({ + # keep track of custom extract command (if any) + 'extract_cmd': src['cmd'], + # copy 'path' entry to 'src' for use with extensions + 'src': src['path'], + }) else: # use default template for name of source file if none is specified diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 27dd812e20..e85b94daec 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -119,6 +119,7 @@ def __init__(self, mself, ext, extra_params=None): # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) + self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index c3b5c7a9fb..3c427c5a9f 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -118,7 +118,7 @@ def run(self, unpack_src=False): if unpack_src: targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options, - change_into_dir=False) + change_into_dir=False, cmd=self.src_extract_cmd) # setting start dir must be done from unpacked source directory for extension, # because start_dir value is usually a relative path (if it is set) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 99546c329d..bde3d1156f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -456,11 +456,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced _log.debug("Unpacking %s in directory %s", fn, abs_dest) cwd = change_dir(abs_dest) - if not cmd: - cmd = extract_cmd(fn, overwrite=overwrite) - else: + if cmd: # complete command template with filename cmd = cmd % fn + _log.debug("Using specified command to unpack %s: %s", fn, cmd) + else: + cmd = extract_cmd(fn, overwrite=overwrite) + _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd) + if not cmd: raise EasyBuildError("Can't extract file %s with unknown filetype", fn) @@ -1366,7 +1369,7 @@ def find_extension(filename): if res: ext = res.group('ext') else: - raise EasyBuildError('Unknown file type for file %s', filename) + raise EasyBuildError("%s has unknown file extension", filename) return ext @@ -1379,7 +1382,9 @@ def extract_cmd(filepath, overwrite=False): ext = find_extension(filename) target = filename[:-len(ext)] + # find_extension will either return an extension listed in EXTRACT_CMDS, or raise an error cmd_tmpl = EXTRACT_CMDS[ext.lower()] + if overwrite: if 'unzip -qq' in cmd_tmpl: cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c54f5881f..57f5d18975 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -114,6 +114,9 @@ def test_extract_cmd(self): self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True)) + error_pattern = "test.foo has unknown file extension" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_cmd, 'test.foo') + def test_find_extension(self): """Test find_extension function.""" tests = [ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index dc3e168e7d..3f46f2f4c3 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1362,6 +1362,31 @@ def test_toy_extension_sources(self): write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) + def test_toy_extension_extract_cmd(self): + """Test for custom extract_cmd specified for an extension.""" + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + toy_ec_txt, + 'exts_list = [', + ' ("bar", "0.0", {', + # deliberately incorrect custom extract command, just to verify that it's picked up + ' "sources": [{', + ' "filename": "bar-%(version)s.tar.gz",', + ' "extract_cmd": "unzip %s",', + ' }],', + ' }),', + ']', + ]) + write_file(test_ec, test_ec_txt) + + error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, + raise_error=True, verbose=False) + def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')