From 055784d2c5f5afdb0d40631c432b350f25fad8d5 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Wed, 22 Jan 2020 16:30:42 +0000 Subject: [PATCH 1/6] Extraced config.guess code from configuremake.py and used in rpackage.py --- easybuild/easyblocks/generic/_config_guess.py | 119 ++++++++++++++++++ easybuild/easyblocks/generic/configuremake.py | 112 +---------------- easybuild/easyblocks/generic/rpackage.py | 20 ++- 3 files changed, 140 insertions(+), 111 deletions(-) create mode 100644 easybuild/easyblocks/generic/_config_guess.py diff --git a/easybuild/easyblocks/generic/_config_guess.py b/easybuild/easyblocks/generic/_config_guess.py new file mode 100644 index 0000000000..b26d6c7c7b --- /dev/null +++ b/easybuild/easyblocks/generic/_config_guess.py @@ -0,0 +1,119 @@ +# This file contains constants and functions required to find and update +# config.guess files in a source tree. + +import os +import re +from datetime import datetime + +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.config import source_paths +from easybuild.tools.filetools import read_file, remove_file, verify_checksum +from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file + +# download location & SHA256 for config.guess script +# note: if this is updated, don't forget to trash the cached download from generic/Configure//! +CONFIG_GUESS_VERSION = '2018-08-29' +CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=" +CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad" +CONFIG_GUESS_SHA256 = "c02eb9cc55c86cfd1e9a794e548d25db5c9539e7b2154beb649bc6e2cbffc74c" + +class ConfigGuessUpdater(EasyBlock): + def __init__(self, *args, **kwargs): + """Initialize easyblock.""" + super(ConfigGuessUpdater, self).__init__(*args, **kwargs) + + self.config_guess = None + + def fetch_step(self, *args, **kwargs): + """Custom fetch step for ConfigGuessUpdater so we use an updated config.guess.""" + super(ConfigGuessUpdater, self).fetch_step(*args, **kwargs) + + # Use an updated config.guess from a global location (if possible) + self.config_guess = self.obtain_config_guess() + + def obtain_config_guess(self, download_source_path=None, search_source_paths=None): + """ + Locate or download an up-to-date config.guess for use with ConfigureMake + + :param download_source_path: Path to download config.guess to + :param search_source_paths: Paths to search for config.guess + :return: Path to config.guess or None + """ + eb_source_paths = source_paths() + if download_source_path is None: + download_source_path = eb_source_paths[0] + if search_source_paths is None: + search_source_paths = eb_source_paths + + config_guess = 'config.guess' + sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') + + config_guess_path = None + + # check if config.guess has already been downloaded to source path + for path in eb_source_paths: + cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) + if os.path.isfile(cand_config_guess_path): + config_guess_path = cand_config_guess_path + self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path) + break + + # if not found, try to download it + if config_guess_path is None: + cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) + config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID + downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path) + if downloaded_path is not None: + # verify SHA256 checksum of download to avoid using a corrupted download + if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256): + config_guess_path = downloaded_path + # add execute permissions + adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) + self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path) + else: + self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path) + remove_file(downloaded_path) + else: + self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)", + config_guess, cand_config_guess_path) + + return config_guess_path + + def check_config_guess(self): + """ + Check timestamp & SHA256 checksum of config.guess script. + + Returns True if ok (or there is no config.guess for this package) and False if it's too old + or doesn't match the checksum. + """ + # log version, timestamp & SHA256 checksum of config.guess that was found (if any) + if self.config_guess: + # config.guess includes a "timestamp='...'" indicating the version + config_guess_version = None + version_regex = re.compile("^timestamp='(.*)'", re.M) + res = version_regex.search(read_file(self.config_guess)) + if res: + config_guess_version = res.group(1) + + config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256) + try: + config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat() + except OSError as err: + self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err) + config_guess_timestamp = None + + self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)", + config_guess_version, config_guess_timestamp, config_guess_checksum) + + if config_guess_version != CONFIG_GUESS_VERSION: + tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION) + self.log.warning("config.guess version at %s does not match expected version: %s vs %s" % tup) + return False + + if config_guess_checksum != CONFIG_GUESS_SHA256: + tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256) + self.log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup) + return False + + return True diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 00fd7973ec..414ff1eeef 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2019 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,38 +36,26 @@ @author: Alan O'Cais (Juelich Supercomputing Centre) """ import os -import re import stat -from datetime import datetime -from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import print_warning -from easybuild.tools.config import source_paths -from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file -from easybuild.tools.filetools import read_file, remove_file, verify_checksum +from easybuild.tools.filetools import read_file from easybuild.tools.run import run_cmd +from easybuild.easyblocks.generic._config_guess import ConfigGuessUpdater # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes # (mainly important when EasyBuild is using Python 3) AUTOCONF_GENERATED_MSG = b"Generated by GNU Autoconf" -# download location & SHA256 for config.guess script -# note: if this is updated, don't forget to trash the cached download from generic/Configure//! -CONFIG_GUESS_VERSION = '2018-08-29' -CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=" -CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad" -CONFIG_GUESS_SHA256 = "c02eb9cc55c86cfd1e9a794e548d25db5c9539e7b2154beb649bc6e2cbffc74c" - - DEFAULT_CONFIGURE_CMD = './configure' DEFAULT_BUILD_CMD = 'make' DEFAULT_INSTALL_CMD = 'make install' -class ConfigureMake(EasyBlock): +class ConfigureMake(ConfigGuessUpdater): """ Support for building and installing applications with configure/make/make install """ @@ -92,96 +80,6 @@ def extra_options(extra_vars=None): }) return extra_vars - def __init__(self, *args, **kwargs): - """Initialize easyblock.""" - super(ConfigureMake, self).__init__(*args, **kwargs) - - self.config_guess = None - - def obtain_config_guess(self, download_source_path=None, search_source_paths=None): - """ - Locate or download an up-to-date config.guess for use with ConfigureMake - - :param download_source_path: Path to download config.guess to - :param search_source_paths: Paths to search for config.guess - :return: Path to config.guess or None - """ - eb_source_paths = source_paths() - if download_source_path is None: - download_source_path = eb_source_paths[0] - if search_source_paths is None: - search_source_paths = eb_source_paths - - config_guess = 'config.guess' - sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') - - config_guess_path = None - - # check if config.guess has already been downloaded to source path - for path in eb_source_paths: - cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) - if os.path.isfile(cand_config_guess_path): - config_guess_path = cand_config_guess_path - self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path) - break - - # if not found, try to download it - if config_guess_path is None: - cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) - config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID - downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path) - if downloaded_path is not None: - # verify SHA256 checksum of download to avoid using a corrupted download - if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256): - config_guess_path = downloaded_path - # add execute permissions - adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) - self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path) - else: - self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path) - remove_file(downloaded_path) - else: - self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)", - config_guess, cand_config_guess_path) - - return config_guess_path - - def check_config_guess(self): - """Check timestamp & SHA256 checksum of config.guess script.""" - # log version, timestamp & SHA256 checksum of config.guess that was found (if any) - if self.config_guess: - # config.guess includes a "timestamp='...'" indicating the version - config_guess_version = None - version_regex = re.compile("^timestamp='(.*)'", re.M) - res = version_regex.search(read_file(self.config_guess)) - if res: - config_guess_version = res.group(1) - - config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256) - try: - config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat() - except OSError as err: - self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err) - config_guess_timestamp = None - - self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)", - config_guess_version, config_guess_timestamp, config_guess_checksum) - - if config_guess_version != CONFIG_GUESS_VERSION: - tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION) - print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup) - - if config_guess_checksum != CONFIG_GUESS_SHA256: - tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256) - print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup) - - def fetch_step(self, *args, **kwargs): - """Custom fetch step for ConfigureMake so we use an updated config.guess.""" - super(ConfigureMake, self).fetch_step(*args, **kwargs) - - # Use an updated config.guess from a global location (if possible) - self.config_guess = self.obtain_config_guess() - def configure_step(self, cmd_prefix=''): """ Configure step @@ -229,7 +127,7 @@ def configure_step(self, cmd_prefix=''): if build_type is None or host_type is None: - # config.guess script may not be obtained yet despite the call in fetch_step, + # config.guess script may not be obtained yet despite the call in fetch_step in ConfigGuessUpdater, # for example when installing a Bundle component with ConfigureMake if self.config_guess is None: self.config_guess = self.obtain_config_guess() diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 1b9775084c..adce79cb07 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2019 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,8 +39,9 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir +from easybuild.tools.filetools import mkdir, copy_file from easybuild.tools.run import run_cmd, parse_log_for_error +from easybuild.easyblocks.generic._config_guess import ConfigGuessUpdater def make_R_install_option(opt, values, cmdline=False): @@ -74,12 +75,12 @@ def extra_options(extra_vars=None): extra_vars.update({ 'exts_subdir': ['', "Subdirectory where R extensions should be installed info", CUSTOM], 'unpack_sources': [False, "Unpack sources before installation", CUSTOM], + 'update_config_guess': [False, "Update the config.guess file(s) in this package", CUSTOM], }) return extra_vars def __init__(self, *args, **kwargs): """Initliaze RPackage-specific class variables.""" - super(RPackage, self).__init__(*args, **kwargs) self.configurevars = [] @@ -215,10 +216,21 @@ def run(self): lib_install_prefix = os.path.join(self.installdir, self.cfg['exts_subdir']) mkdir(lib_install_prefix, parents=True) - if self.patches: + if self.patches or self.cfg['update_config_guess']: super(RPackage, self).run(unpack_src=True) else: super(RPackage, self).run() + + if self.cfg['update_config_guess']: + cgu = ConfigGuessUpdater(self.cfg) + for root, dirs, files in os.walk(self.builddir): + for name in files: + if name == 'config.guess': + cgu.config_guess = os.path.join(root, name) + if not cgu.check_config_guess(): + updated = cgu.obtain_config_guess() + self.log.warning("Using updated config.guess for %s", cgu.config_guess) + copy_file(updated, cgu.config_guess) if self.src: self.ext_src = self.src From 01d18ba94cc71604cef23463d2706fba71028e72 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Wed, 22 Jan 2020 16:37:38 +0000 Subject: [PATCH 2/6] Hound's complaints --- easybuild/easyblocks/generic/_config_guess.py | 5 ++++- easybuild/easyblocks/generic/configuremake.py | 1 - easybuild/easyblocks/generic/rpackage.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/_config_guess.py b/easybuild/easyblocks/generic/_config_guess.py index b26d6c7c7b..c28d9d05ef 100644 --- a/easybuild/easyblocks/generic/_config_guess.py +++ b/easybuild/easyblocks/generic/_config_guess.py @@ -3,6 +3,7 @@ import os import re +import stat from datetime import datetime from easybuild.framework.easyblock import EasyBlock @@ -18,6 +19,7 @@ CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad" CONFIG_GUESS_SHA256 = "c02eb9cc55c86cfd1e9a794e548d25db5c9539e7b2154beb649bc6e2cbffc74c" + class ConfigGuessUpdater(EasyBlock): def __init__(self, *args, **kwargs): """Initialize easyblock.""" @@ -113,7 +115,8 @@ def check_config_guess(self): if config_guess_checksum != CONFIG_GUESS_SHA256: tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256) - self.log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup) + self.log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" + % tup) return False return True diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 414ff1eeef..d6c2b8bb81 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -36,7 +36,6 @@ @author: Alan O'Cais (Juelich Supercomputing Centre) """ import os -import stat from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index adce79cb07..733c529075 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -220,10 +220,10 @@ def run(self): super(RPackage, self).run(unpack_src=True) else: super(RPackage, self).run() - + if self.cfg['update_config_guess']: cgu = ConfigGuessUpdater(self.cfg) - for root, dirs, files in os.walk(self.builddir): + for root, _, files in os.walk(self.builddir): for name in files: if name == 'config.guess': cgu.config_guess = os.path.join(root, name) From 5b75a1176a13eab19bd7a0c77fc1855f6c3c5b63 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Wed, 22 Jan 2020 17:14:56 +0000 Subject: [PATCH 3/6] Making it still install from the extracted directory even with no patches --- easybuild/easyblocks/generic/_config_guess.py | 6 +++--- easybuild/easyblocks/generic/rpackage.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/_config_guess.py b/easybuild/easyblocks/generic/_config_guess.py index c28d9d05ef..145f03f89c 100644 --- a/easybuild/easyblocks/generic/_config_guess.py +++ b/easybuild/easyblocks/generic/_config_guess.py @@ -11,6 +11,7 @@ from easybuild.tools.filetools import read_file, remove_file, verify_checksum from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file +from easybuild.tools.build_log import print_warning # download location & SHA256 for config.guess script # note: if this is updated, don't forget to trash the cached download from generic/Configure//! @@ -110,13 +111,12 @@ def check_config_guess(self): if config_guess_version != CONFIG_GUESS_VERSION: tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION) - self.log.warning("config.guess version at %s does not match expected version: %s vs %s" % tup) + print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup) return False if config_guess_checksum != CONFIG_GUESS_SHA256: tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256) - self.log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" - % tup) + print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup) return False return True diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 733c529075..ab358e4a53 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -42,6 +42,7 @@ from easybuild.tools.filetools import mkdir, copy_file from easybuild.tools.run import run_cmd, parse_log_for_error from easybuild.easyblocks.generic._config_guess import ConfigGuessUpdater +from easybuild.tools.build_log import print_warning def make_R_install_option(opt, values, cmdline=False): @@ -135,7 +136,7 @@ def make_cmdline_cmd(self, prefix=None): if self.cfg['unpack_sources']: loc = self.start_dir - elif self.patches: + elif self.patches or self.cfg['update_config_guess']: loc = self.ext_dir else: loc = self.ext_src @@ -229,7 +230,7 @@ def run(self): cgu.config_guess = os.path.join(root, name) if not cgu.check_config_guess(): updated = cgu.obtain_config_guess() - self.log.warning("Using updated config.guess for %s", cgu.config_guess) + print_warning("Using updated config.guess for %s", cgu.config_guess) copy_file(updated, cgu.config_guess) if self.src: From 5350e655b801cc19647fb1873c06f47e991b0798 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Thu, 23 Jan 2020 08:52:46 +0000 Subject: [PATCH 4/6] Changing to a minimum-sized changeset for config.guess updating in rpackage.py --- easybuild/easyblocks/generic/_config_guess.py | 122 ------------------ easybuild/easyblocks/generic/configuremake.py | 120 ++++++++++++++++- easybuild/easyblocks/generic/rpackage.py | 4 +- 3 files changed, 118 insertions(+), 128 deletions(-) delete mode 100644 easybuild/easyblocks/generic/_config_guess.py diff --git a/easybuild/easyblocks/generic/_config_guess.py b/easybuild/easyblocks/generic/_config_guess.py deleted file mode 100644 index 145f03f89c..0000000000 --- a/easybuild/easyblocks/generic/_config_guess.py +++ /dev/null @@ -1,122 +0,0 @@ -# This file contains constants and functions required to find and update -# config.guess files in a source tree. - -import os -import re -import stat -from datetime import datetime - -from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.config import source_paths -from easybuild.tools.filetools import read_file, remove_file, verify_checksum -from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION -from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file -from easybuild.tools.build_log import print_warning - -# download location & SHA256 for config.guess script -# note: if this is updated, don't forget to trash the cached download from generic/Configure//! -CONFIG_GUESS_VERSION = '2018-08-29' -CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=" -CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad" -CONFIG_GUESS_SHA256 = "c02eb9cc55c86cfd1e9a794e548d25db5c9539e7b2154beb649bc6e2cbffc74c" - - -class ConfigGuessUpdater(EasyBlock): - def __init__(self, *args, **kwargs): - """Initialize easyblock.""" - super(ConfigGuessUpdater, self).__init__(*args, **kwargs) - - self.config_guess = None - - def fetch_step(self, *args, **kwargs): - """Custom fetch step for ConfigGuessUpdater so we use an updated config.guess.""" - super(ConfigGuessUpdater, self).fetch_step(*args, **kwargs) - - # Use an updated config.guess from a global location (if possible) - self.config_guess = self.obtain_config_guess() - - def obtain_config_guess(self, download_source_path=None, search_source_paths=None): - """ - Locate or download an up-to-date config.guess for use with ConfigureMake - - :param download_source_path: Path to download config.guess to - :param search_source_paths: Paths to search for config.guess - :return: Path to config.guess or None - """ - eb_source_paths = source_paths() - if download_source_path is None: - download_source_path = eb_source_paths[0] - if search_source_paths is None: - search_source_paths = eb_source_paths - - config_guess = 'config.guess' - sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') - - config_guess_path = None - - # check if config.guess has already been downloaded to source path - for path in eb_source_paths: - cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) - if os.path.isfile(cand_config_guess_path): - config_guess_path = cand_config_guess_path - self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path) - break - - # if not found, try to download it - if config_guess_path is None: - cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) - config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID - downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path) - if downloaded_path is not None: - # verify SHA256 checksum of download to avoid using a corrupted download - if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256): - config_guess_path = downloaded_path - # add execute permissions - adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) - self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path) - else: - self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path) - remove_file(downloaded_path) - else: - self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)", - config_guess, cand_config_guess_path) - - return config_guess_path - - def check_config_guess(self): - """ - Check timestamp & SHA256 checksum of config.guess script. - - Returns True if ok (or there is no config.guess for this package) and False if it's too old - or doesn't match the checksum. - """ - # log version, timestamp & SHA256 checksum of config.guess that was found (if any) - if self.config_guess: - # config.guess includes a "timestamp='...'" indicating the version - config_guess_version = None - version_regex = re.compile("^timestamp='(.*)'", re.M) - res = version_regex.search(read_file(self.config_guess)) - if res: - config_guess_version = res.group(1) - - config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256) - try: - config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat() - except OSError as err: - self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err) - config_guess_timestamp = None - - self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)", - config_guess_version, config_guess_timestamp, config_guess_checksum) - - if config_guess_version != CONFIG_GUESS_VERSION: - tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION) - print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup) - return False - - if config_guess_checksum != CONFIG_GUESS_SHA256: - tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256) - print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup) - return False - - return True diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index d6c2b8bb81..f2a5727e5f 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -36,25 +36,38 @@ @author: Alan O'Cais (Juelich Supercomputing Centre) """ import os +import re +import stat +from datetime import datetime +from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import print_warning -from easybuild.tools.filetools import read_file +from easybuild.tools.config import source_paths +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file +from easybuild.tools.filetools import read_file, remove_file, verify_checksum from easybuild.tools.run import run_cmd -from easybuild.easyblocks.generic._config_guess import ConfigGuessUpdater # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes # (mainly important when EasyBuild is using Python 3) AUTOCONF_GENERATED_MSG = b"Generated by GNU Autoconf" +# download location & SHA256 for config.guess script +# note: if this is updated, don't forget to trash the cached download from generic/Configure//! +CONFIG_GUESS_VERSION = '2018-08-29' +CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=" +CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad" +CONFIG_GUESS_SHA256 = "c02eb9cc55c86cfd1e9a794e548d25db5c9539e7b2154beb649bc6e2cbffc74c" + + DEFAULT_CONFIGURE_CMD = './configure' DEFAULT_BUILD_CMD = 'make' DEFAULT_INSTALL_CMD = 'make install' -class ConfigureMake(ConfigGuessUpdater): +class ConfigureMake(EasyBlock): """ Support for building and installing applications with configure/make/make install """ @@ -79,6 +92,105 @@ def extra_options(extra_vars=None): }) return extra_vars + def __init__(self, *args, **kwargs): + """Initialize easyblock.""" + super(ConfigGuessUpdater, self).__init__(*args, **kwargs) + + self.config_guess = None + + def obtain_config_guess(self, download_source_path=None, search_source_paths=None): + """ + Locate or download an up-to-date config.guess for use with ConfigureMake + + :param download_source_path: Path to download config.guess to + :param search_source_paths: Paths to search for config.guess + :return: Path to config.guess or None + """ + eb_source_paths = source_paths() + if download_source_path is None: + download_source_path = eb_source_paths[0] + if search_source_paths is None: + search_source_paths = eb_source_paths + + config_guess = 'config.guess' + sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') + + config_guess_path = None + + # check if config.guess has already been downloaded to source path + for path in eb_source_paths: + cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) + if os.path.isfile(cand_config_guess_path): + config_guess_path = cand_config_guess_path + self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path) + break + + # if not found, try to download it + if config_guess_path is None: + cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) + config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID + downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path) + if downloaded_path is not None: + # verify SHA256 checksum of download to avoid using a corrupted download + if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256): + config_guess_path = downloaded_path + # add execute permissions + adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) + self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path) + else: + self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path) + remove_file(downloaded_path) + else: + self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)", + config_guess, cand_config_guess_path) + + return config_guess_path + + def check_config_guess(self): + """ + Check timestamp & SHA256 checksum of config.guess script. + + Returns True if ok (or there is no config.guess for this package) and False if it's too old + or doesn't match the checksum. + """ + # log version, timestamp & SHA256 checksum of config.guess that was found (if any) + if self.config_guess: + # config.guess includes a "timestamp='...'" indicating the version + config_guess_version = None + version_regex = re.compile("^timestamp='(.*)'", re.M) + res = version_regex.search(read_file(self.config_guess)) + if res: + config_guess_version = res.group(1) + + config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256) + try: + config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat() + except OSError as err: + self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err) + config_guess_timestamp = None + + self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)", + config_guess_version, config_guess_timestamp, config_guess_checksum) + + if config_guess_version != CONFIG_GUESS_VERSION: + tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION) + print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup) + return False + + if config_guess_checksum != CONFIG_GUESS_SHA256: + tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256) + print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup) + return False + + return True + + def fetch_step(self, *args, **kwargs): + """Custom fetch step for ConfigGuessUpdater so we use an updated config.guess.""" + super(ConfigGuessUpdater, self).fetch_step(*args, **kwargs) + + # Use an updated config.guess from a global location (if possible) + self.config_guess = self.obtain_config_guess() + def configure_step(self, cmd_prefix=''): """ Configure step @@ -126,7 +238,7 @@ def configure_step(self, cmd_prefix=''): if build_type is None or host_type is None: - # config.guess script may not be obtained yet despite the call in fetch_step in ConfigGuessUpdater, + # config.guess script may not be obtained yet despite the call in fetch_step, # for example when installing a Bundle component with ConfigureMake if self.config_guess is None: self.config_guess = self.obtain_config_guess() diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index ab358e4a53..56537ae43e 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -41,7 +41,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, copy_file from easybuild.tools.run import run_cmd, parse_log_for_error -from easybuild.easyblocks.generic._config_guess import ConfigGuessUpdater +from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import print_warning @@ -223,7 +223,7 @@ def run(self): super(RPackage, self).run() if self.cfg['update_config_guess']: - cgu = ConfigGuessUpdater(self.cfg) + cgu = ConfigureMake(self.cfg) for root, _, files in os.walk(self.builddir): for name in files: if name == 'config.guess': From 5c2983001b0b9ee92ae235b60548e2a093142638 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Thu, 23 Jan 2020 08:55:13 +0000 Subject: [PATCH 5/6] Removing unnecessary changes --- easybuild/easyblocks/generic/configuremake.py | 8 ++++---- easybuild/easyblocks/generic/rpackage.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index f2a5727e5f..ed581ae731 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2019 Ghent University +# Copyright 2009-2020 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -94,7 +94,7 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Initialize easyblock.""" - super(ConfigGuessUpdater, self).__init__(*args, **kwargs) + super(ConfigureMake, self).__init__(*args, **kwargs) self.config_guess = None @@ -185,8 +185,8 @@ def check_config_guess(self): return True def fetch_step(self, *args, **kwargs): - """Custom fetch step for ConfigGuessUpdater so we use an updated config.guess.""" - super(ConfigGuessUpdater, self).fetch_step(*args, **kwargs) + """Custom fetch step for ConfigureMake so we use an updated config.guess.""" + super(ConfigureMake, self).fetch_step(*args, **kwargs) # Use an updated config.guess from a global location (if possible) self.config_guess = self.obtain_config_guess() diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 56537ae43e..e38b09bb19 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2019 Ghent University +# Copyright 2009-2020 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -82,6 +82,7 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Initliaze RPackage-specific class variables.""" + super(RPackage, self).__init__(*args, **kwargs) self.configurevars = [] From 0877c4cf44b44d5b8114b2153d74fb46f4a2cfc8 Mon Sep 17 00:00:00 2001 From: edmondac-admin Date: Thu, 23 Jan 2020 08:56:13 +0000 Subject: [PATCH 6/6] vbl name --- easybuild/easyblocks/generic/rpackage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index e38b09bb19..61f1a55502 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -224,15 +224,15 @@ def run(self): super(RPackage, self).run() if self.cfg['update_config_guess']: - cgu = ConfigureMake(self.cfg) + confmake = ConfigureMake(self.cfg) for root, _, files in os.walk(self.builddir): for name in files: if name == 'config.guess': - cgu.config_guess = os.path.join(root, name) - if not cgu.check_config_guess(): - updated = cgu.obtain_config_guess() - print_warning("Using updated config.guess for %s", cgu.config_guess) - copy_file(updated, cgu.config_guess) + confmake.config_guess = os.path.join(root, name) + if not confmake.check_config_guess(): + updated = confmake.obtain_config_guess() + print_warning("Using updated config.guess for %s", confmake.config_guess) + copy_file(updated, confmake.config_guess) if self.src: self.ext_src = self.src