diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 31d906c1f9..27987fb9a5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -35,6 +35,7 @@ :author: Ward Poelmans (Ghent University) :author: Fotis Georgatos (Uni.Lu, NTUA) :author: Damian Alvarez (Forschungszentrum Juelich GmbH) +:author: Maxime Boissonneault (Compute Canada) """ import copy @@ -71,9 +72,9 @@ from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, is_sha256_checksum, mkdir -from easybuild.tools.filetools import move_logs, read_file, remove_file, rmtree2, verify_checksum, weld_paths -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import encode_class_name, extract_file, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_logs, read_file, remove_file, rmtree2 +from easybuild.tools.filetools import verify_checksum, weld_paths, write_file from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP @@ -339,17 +340,19 @@ def fetch_sources(self, sources=None, checksums=None): checksums = self.cfg['checksums'] for index, source in enumerate(sources): - extract_cmd, download_filename, source_urls = None, None, None + extract_cmd, download_filename, source_urls, git_config = None, None, None, None if isinstance(source, basestring): filename = source elif isinstance(source, dict): + # Making a copy to avoid modifying the object with pops source = source.copy() filename = source.pop('filename', None) extract_cmd = source.pop('extract_cmd', None) download_filename = source.pop('download_filename', None) source_urls = source.pop('source_urls', None) + git_config = source.pop('git_config', None) if source: raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source) @@ -363,7 +366,7 @@ def fetch_sources(self, sources=None, checksums=None): # check if the sources can be located force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] path = self.obtain_file(filename, download_filename=download_filename, force_download=force_download, - urls=source_urls) + urls=source_urls, git_config=git_config) if path: self.log.debug('File %s found for source %s' % (path, filename)) self.src.append({ @@ -562,7 +565,8 @@ def fetch_extension_sources(self, skip_checksums=False): return exts_sources - def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False): + def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, + git_config=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -571,7 +575,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No :param extension: indicates whether locations for extension sources should also be considered :param urls: list of source URLs where this file may be available :param download_filename: filename with which the file should be downloaded, and then renamed to - :force_download: always try to download file, even if it's already available in source path + :param force_download: always try to download file, even if it's already available in source path + :param git_config: dictionary to define how to download a git repository """ srcpaths = source_paths() @@ -662,10 +667,14 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No break # no need to try other source paths + targetdir = os.path.join(srcpaths[0], self.name.lower()[0], self.name) + if foundfile: if self.dry_run: self.dry_run_msg(" * %s found at %s", filename, foundfile) return foundfile + elif git_config: + return get_source_tarball_from_git(filename, targetdir, git_config) else: # try and download source files from specified source URLs if urls: @@ -674,7 +683,6 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - targetdir = os.path.join(srcpaths[0], self.name.lower()[0], self.name) mkdir(targetdir, parents=True) for url in source_urls: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 9fa49310c2..b2fb9186d0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -36,6 +36,7 @@ :author: Sotiris Fragkiskos (NTUA, CERN) :author: Davide Vanzo (ACCRE, Vanderbilt University) :author: Damian Alvarez (Forschungszentrum Juelich GmbH) +:author: Maxime Boissonneault (Compute Canada) """ import datetime import difflib @@ -266,7 +267,41 @@ def remove_file(path): if os.path.exists(path) or os.path.islink(path): os.remove(path) except OSError, err: - raise EasyBuildError("Failed to remove %s: %s", path, err) + raise EasyBuildError("Failed to remove file %s: %s", path, err) + + +def remove_dir(path): + """Remove directory at specified path.""" + # early exit in 'dry run' mode + if build_option('extended_dry_run'): + dry_run_msg("directory %s removed" % path, silent=build_option('silent')) + return + + try: + if os.path.exists(path): + rmtree2(path) + except OSError, err: + raise EasyBuildError("Failed to remove directory %s: %s", path, err) + + +def remove(paths): + """ + Remove single file/directory or list of files and directories + + :param paths: path(s) to remove + """ + if isinstance(paths, basestring): + paths = [paths] + + _log.info("Removing %d files & directories", len(paths)) + + for path in paths: + if os.path.isfile(path): + remove_file(path) + elif os.path.isdir(path): + remove_dir(path) + else: + raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path) def change_dir(path): @@ -1669,6 +1704,83 @@ def copy(paths, target_path, force_in_dry_run=False): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) +def get_source_tarball_from_git(filename, targetdir, git_config): + """ + Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it + + :param filename: name of the archive to save the code to (must be .tar.gz) + :param targetdir: target directory where to save the archive to + :param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit + """ + # sanity check on git_config value being passed + if not isinstance(git_config, dict): + raise EasyBuildError("Found unexpected type of value for 'git_config' argument: %s" % type(git_config)) + + # Making a copy to avoid modifying the object with pops + git_config = git_config.copy() + tag = git_config.pop('tag', None) + url = git_config.pop('url', None) + repo_name = git_config.pop('repo_name', None) + commit = git_config.pop('commit', None) + recursive = git_config.pop('recursive', False) + + # input validation of git_config dict + if git_config: + raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: %s", git_config) + + if not repo_name: + raise EasyBuildError("repo_name not specified in git_config parameter") + + if not tag and not commit: + raise EasyBuildError("Neither tag nor commit found in git_config parameter") + + if tag and commit: + raise EasyBuildError("Tag and commit are mutually exclusive in git_config parameter") + + if not url: + raise EasyBuildError("url not specified in git_config parameter") + + if not filename.endswith('.tar.gz'): + raise EasyBuildError("git_config currently only supports filename ending in .tar.gz") + + # prepare target directory and clone repository + mkdir(targetdir, parents=True) + targetpath = os.path.join(targetdir, filename) + + # compose 'git clone' command, and run it + clone_cmd = ['git', 'clone'] + + if tag: + clone_cmd.extend(['--branch', tag]) + + if recursive: + clone_cmd.append('--recursive') + + clone_cmd.append('%s/%s.git' % (url, repo_name)) + + tmpdir = tempfile.mkdtemp() + cwd = change_dir(tmpdir) + run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + + # if a specific commit is asked for, check it out + if commit: + checkout_cmd = ['git', 'checkout', commit] + if recursive: + checkout_cmd.extend(['&&', 'git', 'submodule', 'update']) + + run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + + # create an archive and delete the git repo directory + tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] + run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + + # cleanup (repo_name dir does not exist in dry run mode) + change_dir(cwd) + remove(tmpdir) + + return targetpath + + def move_file(path, target_path, force_in_dry_run=False): """ Move a file from path to target_path diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 51af659ff8..f18b7980ee 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1413,31 +1413,67 @@ def test_extract_file(self): self.assertTrue(os.path.exists(os.path.join(self.test_prefix, 'toy-0.0', 'toy.source'))) self.assertTrue(os.path.samefile(path, self.test_prefix)) - def test_remove_file(self): - """Test remove_file""" + def test_remove(self): + """Test remove_file, remove_dir and join remove functions.""" testfile = os.path.join(self.test_prefix, 'foo') - ft.write_file(testfile, 'bar') + test_dir = os.path.join(self.test_prefix, 'test123') + + for remove_file_function in (ft.remove_file, ft.remove): + ft.write_file(testfile, 'bar') + self.assertTrue(os.path.exists(testfile)) + remove_file_function(testfile) + self.assertFalse(os.path.exists(testfile)) + + for remove_dir_function in (ft.remove_dir, ft.remove): + ft.mkdir(test_dir) + self.assertTrue(os.path.exists(test_dir) and os.path.isdir(test_dir)) + remove_dir_function(test_dir) + self.assertFalse(os.path.exists(test_dir) or os.path.isdir(test_dir)) + # remove also takes a list of paths + ft.write_file(testfile, 'bar') + ft.mkdir(test_dir) self.assertTrue(os.path.exists(testfile)) - ft.remove_file(testfile) + self.assertTrue(os.path.exists(test_dir) and os.path.isdir(test_dir)) + ft.remove([testfile, test_dir]) + self.assertFalse(os.path.exists(testfile)) + self.assertFalse(os.path.exists(test_dir) or os.path.isdir(test_dir)) + # check error handling (after creating a permission problem with removing files/dirs) ft.write_file(testfile, 'bar') + ft.mkdir(test_dir) ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False) self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile) + self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove, testfile) + self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_dir, test_dir) + self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove, test_dir) - # also test behaviour of remove_file under --dry-run + # also test behaviour under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) - self.mock_stdout(True) - ft.remove_file(testfile) - txt = self.get_stdout() - self.mock_stdout(False) - regex = re.compile("^file [^ ]* removed$") - self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + for remove_file_function in (ft.remove_file, ft.remove): + self.mock_stdout(True) + remove_file_function(testfile) + txt = self.get_stdout() + self.mock_stdout(False) + + regex = re.compile("^file [^ ]* removed$") + self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + + for remove_dir_function in (ft.remove_dir, ft.remove): + self.mock_stdout(True) + remove_dir_function(test_dir) + txt = self.get_stdout() + self.mock_stdout(False) + + regex = re.compile("^directory [^ ]* removed$") + self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + + ft.adjust_permissions(self.test_prefix, stat.S_IWUSR, add=True) def test_search_file(self): """Test search_file function.""" @@ -1602,6 +1638,133 @@ def test_diff_files(self): regex = re.compile('^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M) self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res)) + def test_get_source_tarball_from_git(self): + """Test get_source_tarball_from_git function.""" + + git_config = { + 'repo_name': 'testrepository', + 'url': 'https://github.com/hpcugent', + 'tag': 'master', + } + target_dir = os.path.join(self.test_prefix, 'target') + + try: + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + # (only) tarball is created in specified target dir + self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test.tar.gz'))) + self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + + del git_config['tag'] + git_config['commit'] = '8456f86' + res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) + self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test2.tar.gz'))) + self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) + + except EasyBuildError as err: + if "Network is down" in str(err): + print "Ignoring download error in test_get_source_tarball_from_git, working offline?" + else: + raise err + + git_config = { + 'repo_name': 'testrepository', + 'url': 'git@github.com:hpcugent', + 'tag': 'master', + } + args = ['test.tar.gz', self.test_prefix, git_config] + + for key in ['repo_name', 'url', 'tag']: + orig_value = git_config.pop(key) + if key == 'tag': + error_pattern = "Neither tag nor commit found in git_config parameter" + else: + error_pattern = "%s not specified in git_config parameter" % key + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + git_config[key] = orig_value + + git_config['commit'] = '8456f86' + error_pattern = "Tag and commit are mutually exclusive in git_config parameter" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['commit'] + + git_config['unknown'] = 'foobar' + error_pattern = "Found one or more unexpected keys in 'git_config' specification" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['unknown'] + + args[0] = 'test.txt' + error_pattern = "git_config currently only supports filename ending in .tar.gz" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + args[0] = 'test.tar.gz' + + # only test in dry run mode, i.e. check which commands would be executed without actually running them + build_options = { + 'extended_dry_run': True, + 'silent': False, + } + init_config(build_options=build_options) + + def run_check(): + """Helper function to run get_source_tarball_from_git & check dry run output""" + self.mock_stdout(True) + self.mock_stderr(True) + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stderr, '') + regex = re.compile(expected) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.path.dirname(res), target_dir) + self.assertEqual(os.path.basename(res), 'test.tar.gz') + + git_config = { + 'repo_name': 'testrepository', + 'url': 'git@github.com:hpcugent', + 'tag': 'master', + } + expected = '\n'.join([ + ' running command "git clone --branch master git@github.com:hpcugent/testrepository.git"', + " \(in .*/tmp.*\)", + ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + " \(in .*/tmp.*\)", + ]) + run_check() + + git_config['recursive'] = True + expected = '\n'.join([ + ' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', + " \(in .*/tmp.*\)", + ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + " \(in .*/tmp.*\)", + ]) + run_check() + + del git_config['tag'] + git_config['commit'] = '8456f86' + expected = '\n'.join([ + ' running command "git clone --recursive git@github.com:hpcugent/testrepository.git"', + " \(in .*/tmp.*\)", + ' running command "git checkout 8456f86 && git submodule update"', + " \(in testrepository\)", + ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + " \(in .*/tmp.*\)", + ]) + run_check() + + del git_config['recursive'] + expected = '\n'.join([ + ' running command "git clone git@github.com:hpcugent/testrepository.git"', + " \(in .*/tmp.*\)", + ' running command "git checkout 8456f86"', + " \(in testrepository\)", + ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + " \(in .*/tmp.*\)", + ]) + run_check() + def test_is_sha256_checksum(self): """Test for is_sha256_checksum function.""" a_sha256_checksum = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'