diff --git a/.ci/appveyor/install.bat b/.ci/appveyor/install.bat index 15a61ee8036..9548bc0e2c9 100644 --- a/.ci/appveyor/install.bat +++ b/.ci/appveyor/install.bat @@ -1,10 +1,20 @@ if not exist "C:\mingw64" appveyor DownloadFile "https://s3-eu-west-1.amazonaws.com/downloads.conan.io/x86_64-6.3.0-release-posix-sjlj-rt_v5-rev1.7z" if not exist "C:\mingw64" 7z x x86_64-6.3.0-release-posix-sjlj-rt_v5-rev1.7z -oc:\ + +set CMAKE_URL="https://cmake.org/files/v3.7/cmake-3.7.0-win64-x64.zip" +mkdir C:\projects\deps +SET ORIGINAL_DIR=%CD% +cd C:\projects\deps +appveyor DownloadFile %CMAKE_URL% -FileName cmake.zip +7z x cmake.zip -oC:\projects\deps > nul +move C:\projects\deps\cmake-* C:\projects\deps\cmake +set PATH=C:\projects\deps\cmake\bin;%PATH% +cmake --version +cd %ORIGINAL_DIR% + SET PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\mingw64\\bin;%PATH% SET PYTHONPATH=%PYTHONPATH%;%CD% SET CONAN_LOGGING_LEVEL=10 -SET CONAN_COMPILER=Visual Studio -SET CONAN_COMPILER_VERSION=12 %PYTHON%/Scripts/pip.exe install -r conans/requirements.txt %PYTHON%/Scripts/pip.exe install -r conans/requirements_dev.txt %PYTHON%/Scripts/pip.exe install -r conans/requirements_server.txt diff --git a/.ci/appveyor/test.bat b/.ci/appveyor/test.bat index c501810b452..d61badbf253 100644 --- a/.ci/appveyor/test.bat +++ b/.ci/appveyor/test.bat @@ -1 +1 @@ -nosetests --with-coverage conans.test +nosetests --with-coverage --verbosity=2 conans.test --processes=4 --process-timeout=1000 diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 15efea3a88f..2c6c337c163 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -10,4 +10,4 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv activate conan fi -nosetests --with-coverage conans.test +nosetests --with-coverage conans.test --verbosity=2 --processes=4 --process-timeout=1000 diff --git a/.travis.yml b/.travis.yml index 3727ab30fca..c23914bc946 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,21 @@ -language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 os: linux sudo: required dist: trusty +language: python matrix: include: + - python: 2.7 + - python: 3.4 + if: branch =~ (^release.*)|(^master) + - python: 3.5 + if: branch =~ (^release.*)|(^master) + - python: 3.6 - language: generic os: osx osx_image: xcode8.3 env: PYVER=py27 - + if: branch =~ (^release.*)|(^master) - language: generic os: osx osx_image: xcode8.3 @@ -25,6 +26,7 @@ install: - ./.ci/travis/install.sh before_script: - export PYTHONPATH=$PYTHONPATH:$(pwd) + # command to run tests script: - ulimit -n 2048 # Error with py3 and OSX, max file descriptors diff --git a/conans/__init__.py b/conans/__init__.py index 3afd1d90f6c..fc9d70be955 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -16,4 +16,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.26.1' +__version__ = '0.27.0' diff --git a/conans/client/client_cache.py b/conans/client/client_cache.py index 7b78f0eeac5..73219dd7032 100644 --- a/conans/client/client_cache.py +++ b/conans/client/client_cache.py @@ -65,15 +65,11 @@ def registry(self): @property def conan_config(self): - def generate_default_config_file(): - save(self.conan_conf_path, normalize(default_client_conf)) - if not self._conan_config: if not os.path.exists(self.conan_conf_path): - generate_default_config_file() + save(self.conan_conf_path, normalize(default_client_conf)) self._conan_config = ConanClientConfigParser(self.conan_conf_path) - return self._conan_config @property diff --git a/conans/client/cmake.py b/conans/client/cmake.py index eaf556cc597..646d9e37398 100644 --- a/conans/client/cmake.py +++ b/conans/client/cmake.py @@ -39,13 +39,14 @@ def _get_env_cmake_system_name(): class CMake(object): def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True, - parallel=True): + parallel=True, build_type=None): """ :param settings_or_conanfile: Conanfile instance (or settings for retro compatibility) :param generator: Generator name to use or none to autodetect :param cmake_system_name: False to not use CMAKE_SYSTEM_NAME variable, True for auto-detect or directly a string with the system name :param parallel: Try to build with multiple cores if available + :param build_type: Overrides default build type comming from settings """ if isinstance(settings_or_conanfile, Settings): self._settings = settings_or_conanfile @@ -64,10 +65,10 @@ def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True self._compiler = self._settings.get_safe("compiler") self._compiler_version = self._settings.get_safe("compiler.version") self._arch = self._settings.get_safe("arch") - self._build_type = self._settings.get_safe("build_type") self._op_system_version = self._settings.get_safe("os.version") self._libcxx = self._settings.get_safe("compiler.libcxx") self._runtime = self._settings.get_safe("compiler.runtime") + self._build_type = self._settings.get_safe("build_type") self.generator = generator or self._generator() self.build_dir = None @@ -76,6 +77,23 @@ def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True self._cmake_system_name = cmake_system_name self.parallel = parallel self.definitions = self._get_cmake_definitions() + if build_type and build_type != self._build_type: + # Call the setter to warn and update the definitions if needed + self.build_type = build_type + + @property + def build_type(self): + return self._build_type + + @build_type.setter + def build_type(self, build_type): + settings_build_type = self._settings.get_safe("build_type") + if build_type != settings_build_type: + self._conanfile.output.warn( + 'Set CMake build type "%s" is different than the settings build_type "%s"' + % (build_type, settings_build_type)) + self._build_type = build_type + self.definitions.update(self._build_type_definition()) @property def flags(self): @@ -213,10 +231,6 @@ def command_line(self): '-Wno-dev' ]) - @property - def build_type(self): - return self._defs_to_string(self._build_type_definition()) - def _build_type_definition(self): if self._build_type and not self.is_multi_configuration: return {'CMAKE_BUILD_TYPE': self._build_type} @@ -224,7 +238,7 @@ def _build_type_definition(self): @property def runtime(self): - return self._defs_to_string(self._runtime_definition()) + return _defs_to_string(self._runtime_definition()) def _runtime_definition(self): if self._runtime: diff --git a/conans/client/command.py b/conans/client/command.py index 6f09f9cd504..b6a4aca99e2 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -268,7 +268,7 @@ def install(self, *args): filename=args.file, cwd=args.cwd) def config(self, *args): - """Manages conan.conf information + """Manages conan configuration information """ parser = argparse.ArgumentParser(description=self.config.__doc__, prog="conan config") @@ -276,11 +276,13 @@ def config(self, *args): rm_subparser = subparsers.add_parser('rm', help='rm an existing config element') set_subparser = subparsers.add_parser('set', help='set/add value') get_subparser = subparsers.add_parser('get', help='get the value of existing element') + install_subparser = subparsers.add_parser('install', + help='install a full configuration from a zip file, local or remote') rm_subparser.add_argument("item", help="item to remove") get_subparser.add_argument("item", nargs="?", help="item to print") set_subparser.add_argument("item", help="key=value to set") - + install_subparser.add_argument("item", nargs="?", help="configuration file to use") args = parser.parse_args(*args) if args.subcommand == "set": @@ -293,6 +295,8 @@ def config(self, *args): return self._conan.config_get(args.item) elif args.subcommand == "rm": return self._conan.config_rm(args.item) + elif args.subcommand == "install": + return self._conan.config_install(args.item) def info(self, *args): """Prints information about a package recipe's dependency graph. @@ -368,11 +372,11 @@ def info(self, *args): only = [] if only and args.paths and (set(only) - set(path_only_options)): raise ConanException("Invalid --only value '%s' with --path specified, allowed values: [%s]." - % (only, str_path_only_options)) + % (only, str_path_only_options)) elif only and not args.paths and (set(only) - set(info_only_options)): raise ConanException("Invalid --only value '%s', allowed values: [%s].\n" - "Use --only=None to show only the references." % - (only, str_only_options)) + "Use --only=None to show only the references." + % (only, str_only_options)) if args.graph: self._outputer.info_graph(args.graph, deps_graph, project_reference, args.cwd) @@ -451,9 +455,10 @@ def source(self, *args): " folder, then the execution and retrieval of the source code." " Otherwise, if the code has already been retrieved, it will" " do nothing.") + parser.add_argument("--cwd", "-c", help='Use this directory as the current directory') args = parser.parse_args(*args) - return self._conan.source(args.reference, args.force) + return self._conan.source(args.reference, args.force, cwd=args.cwd) def imports(self, *args): """ Execute the 'imports' stage of a conanfile.txt or a conanfile.py. @@ -712,7 +717,6 @@ def remote(self, *args): verify_ssl = get_bool_from_text(args.verify_ssl) if hasattr(args, 'verify_ssl') else False - remote = args.remote if hasattr(args, 'remote') else None url = args.url if hasattr(args, 'url') else None @@ -761,6 +765,10 @@ def profile(self, *args): parser_update.add_argument('item', help='key="value to set", e.j: settings.compiler=gcc') parser_update.add_argument('profile', help='name of the profile') + parser_get = subparsers.add_parser('get', help='Get a profile key') + parser_get.add_argument('item', help='key="value to get", e.j: settings.compiler') + parser_get.add_argument('profile', help='name of the profile') + parser_remove = subparsers.add_parser('remove', help='Remove a profile key') parser_remove.add_argument('item', help='key", e.j: settings.compiler') parser_remove.add_argument('profile', help='name of the profile') @@ -783,6 +791,9 @@ def profile(self, *args): except: raise ConanException("Please specify key=value") self._conan.update_profile(profile, key, value) + elif args.subcommand == "get": + key = args.item + self._outputer.writeln(self._conan.get_profile_key(profile, key)) elif args.subcommand == "remove": self._conan.delete_profile_key(profile, args.item) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 33e8c8d2806..819729aaea4 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -43,7 +43,13 @@ def get_basic_requester(client_cache): requester = requests.Session() - requester.proxies = client_cache.conan_config.proxies + proxies = client_cache.conan_config.proxies + if proxies: + # Account for the requests NO_PROXY env variable, not defined as a proxy like http= + no_proxy = proxies.pop("no_proxy", None) + if no_proxy: + os.environ["NO_PROXY"] = no_proxy + requester.proxies = proxies return requester @@ -168,8 +174,8 @@ def new(self, name, header=False, pure_c=False, test=False, exports_sources=Fals @api_method def test_package(self, profile_name=None, settings=None, options=None, env=None, scope=None, test_folder=None, not_export=False, build=None, keep_source=False, - verify=default_manifest_folder, manifests=default_manifest_folder, - manifests_interactive=default_manifest_folder, + verify=None, manifests=None, + manifests_interactive=None, remote=None, update=False, cwd=None, user=None, channel=None, name=None, version=None): settings = settings or [] @@ -260,8 +266,8 @@ def test_package(self, profile_name=None, settings=None, options=None, env=None, @api_method def create(self, profile_name=None, settings=None, options=None, env=None, scope=None, test_folder=None, not_export=False, build=None, - keep_source=False, verify=default_manifest_folder, - manifests=default_manifest_folder, manifests_interactive=default_manifest_folder, + keep_source=False, verify=None, + manifests=None, manifests_interactive=None, remote=None, update=False, cwd=None, user=None, channel=None, name=None, version=None): @@ -359,8 +365,8 @@ def package_files(self, reference, source_folder=None, build_folder=None, packag @api_method def install(self, reference="", package=None, settings=None, options=None, env=None, scope=None, all=False, - remote=None, werror=False, verify=default_manifest_folder, manifests=default_manifest_folder, - manifests_interactive=default_manifest_folder, build=None, profile_name=None, + remote=None, werror=False, verify=None, manifests=None, + manifests_interactive=None, build=None, profile_name=None, update=False, generator=None, no_imports=False, filename=None, cwd=None): self._user_io.out.werror_active = werror @@ -414,6 +420,11 @@ def config_rm(self, item): config_parser = ConanClientConfigParser(self._client_cache.conan_conf_path) config_parser.rm_item(item) + @api_method + def config_install(self, item): + from conans.client.conf.config_installer import configuration_install + return configuration_install(item, self._client_cache, self._user_io.out, self._runner) + @api_method def info_build_order(self, reference, settings=None, options=None, env=None, scope=None, profile_name=None, filename=None, remote=None, build_order=None, check_updates=None, cwd=None): @@ -526,7 +537,6 @@ def imports(self, reference, undo=False, dest=None, filename=None, cwd=None): current_path = reference self._manager.imports_undo(current_path) else: - cwd = prepare_cwd(cwd) current_path, reference = _get_reference(reference, cwd) self._manager.imports(current_path, reference, filename, dest) @@ -686,6 +696,26 @@ def update_profile(self, profile_name, key, value): profile_path = get_profile_path(profile_name, self._client_cache.profiles_path, os.getcwd()) save(profile_path, contents) + @api_method + def get_profile_key(self, profile_name, key): + first_key, rest_key = self._get_profile_keys(key) + profile, _ = read_profile(profile_name, os.getcwd(), self._client_cache.profiles_path) + try: + if first_key == "settings": + return profile.settings[rest_key] + elif first_key == "options": + return dict(profile.options.as_list())[rest_key] + elif first_key == "env": + package = None + var = rest_key + if ":" in rest_key: + package, var = rest_key.split(":") + return profile.env_values.data[package][var] + elif first_key == "build_requires": + raise ConanException("List the profile manually to see the build_requires") + except KeyError: + raise ConanException("Key not found: '%s'" % key) + @api_method def delete_profile_key(self, profile_name, key): first_key, rest_key = self._get_profile_keys(key) @@ -759,6 +789,8 @@ def _parse_manifests_arguments(verify, manifests, manifests_interactive, cwd): manifest_folder = verify or manifests or manifests_interactive if manifest_folder: if not os.path.isabs(manifest_folder): + if not cwd: + raise ConanException("'cwd' should be defined if the manifest folder is relative.") manifest_folder = os.path.join(cwd, manifest_folder) manifest_verify = verify is not None manifest_interactive = manifests_interactive is not None diff --git a/conans/client/conan_command_output.py b/conans/client/conan_command_output.py index 169d030a3cd..a8100212935 100644 --- a/conans/client/conan_command_output.py +++ b/conans/client/conan_command_output.py @@ -14,6 +14,9 @@ def __init__(self, user_io, client_cache): self.user_io = user_io self.client_cache = client_cache + def writeln(self, value): + self.user_io.out.writeln(value) + def print_profile(self, profile, profile_text): Printer(self.user_io.out).print_profile(profile, profile_text) diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 14dbfb5590e..c52a9ed0062 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -1,11 +1,11 @@ import os -import urllib from six.moves.configparser import ConfigParser, NoSectionError +from six.moves import urllib from conans.errors import ConanException from conans.model.env_info import unquote -from conans.paths import conan_expand_user, DEFAULT_PROFILE_NAME, get_conan_user_home +from conans.paths import conan_expand_user, DEFAULT_PROFILE_NAME from conans.util.env_reader import get_env from conans.util.files import load @@ -60,8 +60,11 @@ default_profile = %s compression_level = 9 # environment CONAN_COMPRESSION_LEVEL sysrequires_sudo = True # environment CONAN_SYSREQUIRES_SUDO +# verbose_traceback = False # environment CONAN_VERBOSE_TRACEBACK # bash_path = "" # environment CONAN_BASH_PATH (only windows) # recipe_linter = False # environment CONAN_RECIPE_LINTER +# pylintrc = path/to/pylintrc_file # environment CONAN_PYLINTRC + # cmake_generator # environment CONAN_CMAKE_GENERATOR # http://www.vtk.org/Wiki/CMake_Cross_Compiling @@ -121,6 +124,8 @@ def env_vars(self): "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_RECIPE_LINTER": self._env_c("general.recipe_linter", "CONAN_RECIPE_LINTER", "True"), "CONAN_CPU_COUNT": self._env_c("general.cpu_count", "CONAN_CPU_COUNT", None), + "CONAN_USER_HOME_SHORT": self._env_c("general.user_home_short", "CONAN_USER_HOME_SHORT", None), + "CONAN_VERBOSE_TRACEBACK": self._env_c("general.verbose_traceback", "CONAN_VERBOSE_TRACEBACK", None), # http://www.vtk.org/Wiki/CMake_Cross_Compiling "CONAN_CMAKE_GENERATOR": self._env_c("general.cmake_generator", "CONAN_CMAKE_GENERATOR", None), "CONAN_CMAKE_TOOLCHAIN_FILE": self._env_c("general.cmake_toolchain_file", "CONAN_CMAKE_TOOLCHAIN_FILE", None), @@ -145,6 +150,7 @@ def env_vars(self): "CONAN_BASH_PATH": self._env_c("general.bash_path", "CONAN_BASH_PATH", None), } + # Filter None values return {name: value for name, value in ret.items() if value is not None} @@ -268,7 +274,8 @@ def proxies(self): proxies = self.get_conf("proxies") # If there is proxies section, but empty, it will try to use system proxy if not proxies: - return urllib.getproxies() - return dict(proxies) + return urllib.request.getproxies() + result = {k: (None if v == "None" else v) for k, v in proxies} + return result except: return None diff --git a/conans/client/conf/config_installer.py b/conans/client/conf/config_installer.py new file mode 100644 index 00000000000..b31e8ef9f4f --- /dev/null +++ b/conans/client/conf/config_installer.py @@ -0,0 +1,108 @@ +import os +from conans.tools import unzip +import shutil +from conans.util.files import rmdir, mkdir +from conans.client.remote_registry import RemoteRegistry +from conans import tools +from conans.errors import ConanException + + +def _handle_remotes(registry_path, remote_file, output): + registry = RemoteRegistry(registry_path, output) + new_registry = RemoteRegistry(remote_file, output) + registry.define_remotes(new_registry.remotes) + + +def _handle_profiles(source_folder, target_folder, output): + mkdir(target_folder) + for root, _, files in os.walk(source_folder): + relative_path = os.path.relpath(root, source_folder) + if relative_path == ".": + relative_path = "" + for f in files: + profile = os.path.join(relative_path, f) + output.info(" Installing profile %s" % profile) + shutil.copy(os.path.join(root, f), os.path.join(target_folder, profile)) + + +def _process_git_repo(repo_url, client_cache, output, runner, tmp_folder): + output.info("Trying to clone repo %s" % repo_url) + + with tools.chdir(tmp_folder): + runner('git clone "%s" config' % repo_url, output=output) + tmp_folder = os.path.join(tmp_folder, "config") + _process_folder(tmp_folder, client_cache, output) + + +def _process_zip_file(zippath, client_cache, output, tmp_folder): + unzip(zippath, tmp_folder) + os.unlink(zippath) + _process_folder(tmp_folder, client_cache, output) + + +def _handle_conan_conf(current_conan_conf, new_conan_conf_path): + current_conan_conf.read(new_conan_conf_path) + with open(current_conan_conf.filename, "w") as f: + current_conan_conf.write(f) + + +def _process_folder(folder, client_cache, output): + for root, dirs, files in os.walk(folder): + for f in files: + if f == "settings.yml": + output.info("Installing settings.yml") + settings_path = client_cache.settings_path + shutil.copy(os.path.join(root, f), settings_path) + elif f == "conan.conf": + output.info("Processing conan.conf") + conan_conf = client_cache.conan_config + _handle_conan_conf(conan_conf, os.path.join(root, f)) + elif f == "remotes.txt": + output.info("Defining remotes") + registry_path = client_cache.registry + _handle_remotes(registry_path, os.path.join(root, f), output) + else: + output.info("Copying file %s to %s" % (f, client_cache.conan_folder)) + shutil.copy(os.path.join(root, f), client_cache.conan_folder) + for d in dirs: + if d == "profiles": + output.info("Installing profiles") + profiles_path = client_cache.profiles_path + _handle_profiles(os.path.join(root, d), profiles_path, output) + break + dirs[:] = [d for d in dirs if d not in ("profiles", ".git")] + + +def _process_download(item, client_cache, output, tmp_folder): + output.info("Trying to download %s" % item) + zippath = os.path.join(tmp_folder, "config.zip") + tools.download(item, zippath, out=output) + _process_zip_file(zippath, client_cache, output, tmp_folder) + + +def configuration_install(item, client_cache, output, runner): + tmp_folder = os.path.join(client_cache.conan_folder, "tmp_config_install") + # necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/ + tmp_folder = os.path.realpath(tmp_folder) + mkdir(tmp_folder) + try: + if item is None: + try: + item = client_cache.conan_config.get_item("general.config_install") + except ConanException: + raise ConanException("Called config install without arguments and " + "'general.config_install' not defined in conan.conf") + + if item.endswith(".git"): + _process_git_repo(item, client_cache, output, runner, tmp_folder) + elif os.path.exists(item): + # is a local file + _process_zip_file(item, client_cache, output, tmp_folder) + elif item.startswith("http"): + _process_download(item, client_cache, output, tmp_folder) + else: + raise ConanException("I don't know how to process %s" % item) + finally: + if item: + client_cache.conan_config.set_item("general.config_install", item) + rmdir(tmp_folder) diff --git a/conans/client/deps_builder.py b/conans/client/deps_builder.py index 6bad0b09d07..3c0143ed1da 100644 --- a/conans/client/deps_builder.py +++ b/conans/client/deps_builder.py @@ -118,13 +118,10 @@ def propagate_info(self): conanfile.build_requires_options = conanfile.options.values conanfile.options.clear_unused(indirect_reqs.union(direct_reqs)) - non_devs = self.non_dev_nodes(node) - conanfile.info = ConanInfo.create(conanfile.settings.values, conanfile.options.values, direct_reqs, - indirect_reqs, - non_devs) + indirect_reqs) # Once we are done, call package_id() to narrow and change possible values if hasattr(conanfile, "conan_info"): @@ -249,28 +246,6 @@ def private_nodes(self, built_private_nodes): result.append(node) return result - def non_dev_nodes(self, root): - if not root.conanfile.scope.dev: - # Optimization. This allow not to check it for most packages, which dev=False - return None - open_nodes = set([root]) - result = set() - expanded = set() - while open_nodes: - new_open_nodes = set() - for node in open_nodes: - neighbors = self._neighbors[node] - requires = node.conanfile.requires - for n in neighbors: - requirement = requires[n.conan_ref.name] - if not requirement.dev and n not in expanded: - result.add(n.conan_ref.name) - new_open_nodes.add(n) - expanded.add(n) - - open_nodes = new_open_nodes - return result - class DepsGraphBuilder(object): """ Responsible for computing the dependencies graph DepsGraph diff --git a/conans/client/export.py b/conans/client/export.py index 7cbe66326ce..29f9e7db6d2 100644 --- a/conans/client/export.py +++ b/conans/client/export.py @@ -4,8 +4,8 @@ import shutil import os -from conans.util.files import save, load, rmdir -from conans.paths import CONAN_MANIFEST, CONANFILE, DIRTY_FILE +from conans.util.files import save, load, rmdir, is_dirty, set_dirty +from conans.paths import CONAN_MANIFEST, CONANFILE from conans.errors import ConanException from conans.model.manifest import FileTreeManifest from conans.client.output import ScopedOutput @@ -73,9 +73,8 @@ def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_so save(os.path.join(destination_folder, CONAN_MANIFEST), str(digest)) source = paths.source(conan_ref, conanfile.short_paths) - dirty = os.path.join(source, DIRTY_FILE) remove = False - if os.path.exists(dirty): + if is_dirty(source): output.info("Source folder is dirty, forcing removal") remove = True elif modified_recipe and not keep_source and os.path.exists(source): @@ -91,7 +90,7 @@ def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_so output.error("Unable to delete source folder. " "Will be marked as dirty for deletion") output.warn(str(e)) - save(os.path.join(source, DIRTY_FILE), "") + set_dirty(source) def _init_export_folder(destination_folder, destination_src_folder): diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index c4306b5c568..15f4404b407 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -1,8 +1,9 @@ -from conans.client.generators.virtualrunenv import VirtualRunEnvGenerator +from os.path import join + from conans.errors import ConanException -from conans.model import registered_generators from conans.util.files import save, normalize -from os.path import join + +from .virtualrunenv import VirtualRunEnvGenerator from .text import TXTGenerator from .gcc import GCCGenerator from .cmake import CMakeGenerator @@ -10,6 +11,7 @@ from .qbs import QbsGenerator from .scons import SConsGenerator from .visualstudio import VisualStudioGenerator +from .visualstudiolegacy import VisualStudioLegacyGenerator from .xcode import XCodeGenerator from .ycm import YouCompleteMeGenerator from .virtualenv import VirtualEnvGenerator @@ -18,24 +20,43 @@ from .virtualbuildenv import VirtualBuildEnvGenerator -def _save_generator(name, klass): - if name not in registered_generators: - registered_generators.add(name, klass) +class _GeneratorManager(object): + def __init__(self): + self._generators = {} + + def add(self, name, generator_class): + if name not in self._generators: + self._generators[name] = generator_class + + @property + def available(self): + return list(self._generators.keys()) + + def __contains__(self, name): + return name in self._generators + + def __getitem__(self, key): + return self._generators[key] + + +registered_generators = _GeneratorManager() + -_save_generator("txt", TXTGenerator) -_save_generator("gcc", GCCGenerator) -_save_generator("cmake", CMakeGenerator) -_save_generator("cmake_multi", CMakeMultiGenerator) -_save_generator("qmake", QmakeGenerator) -_save_generator("qbs", QbsGenerator) -_save_generator("scons", SConsGenerator) -_save_generator("visual_studio", VisualStudioGenerator) -_save_generator("xcode", XCodeGenerator) -_save_generator("ycm", YouCompleteMeGenerator) -_save_generator("virtualenv", VirtualEnvGenerator) -_save_generator("env", ConanEnvGenerator) -_save_generator("virtualbuildenv", VirtualBuildEnvGenerator) -_save_generator("virtualrunenv", VirtualRunEnvGenerator) +registered_generators.add("txt", TXTGenerator) +registered_generators.add("gcc", GCCGenerator) +registered_generators.add("cmake", CMakeGenerator) +registered_generators.add("cmake_multi", CMakeMultiGenerator) +registered_generators.add("qmake", QmakeGenerator) +registered_generators.add("qbs", QbsGenerator) +registered_generators.add("scons", SConsGenerator) +registered_generators.add("visual_studio", VisualStudioGenerator) +registered_generators.add("visual_studio_legacy", VisualStudioLegacyGenerator) +registered_generators.add("xcode", XCodeGenerator) +registered_generators.add("ycm", YouCompleteMeGenerator) +registered_generators.add("virtualenv", VirtualEnvGenerator) +registered_generators.add("env", ConanEnvGenerator) +registered_generators.add("virtualbuildenv", VirtualBuildEnvGenerator) +registered_generators.add("virtualrunenv", VirtualRunEnvGenerator) def write_generators(conanfile, path, output): diff --git a/conans/client/generators/cmake.py b/conans/client/generators/cmake.py index c9712b3505d..07e007aa302 100644 --- a/conans/client/generators/cmake.py +++ b/conans/client/generators/cmake.py @@ -1,4 +1,5 @@ from conans.model import Generator +from conans.model.build_info import CppInfo from conans.paths import BUILD_INFO_CMAKE from conans.client.generators.cmake_common import cmake_dependency_vars,\ cmake_macros, generate_targets_section, cmake_dependencies, cmake_package_info,\ @@ -6,35 +7,34 @@ class DepsCppCmake(object): - def __init__(self, deps_cpp_info): - + def __init__(self, cpp_info): def multiline(field): return "\n\t\t\t".join('"%s"' % p.replace("\\", "/") for p in field) - self.include_paths = multiline(deps_cpp_info.include_paths) - self.lib_paths = multiline(deps_cpp_info.lib_paths) - self.res_paths = multiline(deps_cpp_info.res_paths) - self.bin_paths = multiline(deps_cpp_info.bin_paths) - self.build_paths = multiline(deps_cpp_info.build_paths) + self.include_paths = multiline(cpp_info.include_paths) + self.lib_paths = multiline(cpp_info.lib_paths) + self.res_paths = multiline(cpp_info.res_paths) + self.bin_paths = multiline(cpp_info.bin_paths) + self.build_paths = multiline(cpp_info.build_paths) - self.libs = " ".join(deps_cpp_info.libs) - self.defines = "\n\t\t\t".join("-D%s" % d for d in deps_cpp_info.defines) - self.compile_definitions = "\n\t\t\t".join(deps_cpp_info.defines) + self.libs = " ".join(cpp_info.libs) + self.defines = "\n\t\t\t".join("-D%s" % d for d in cpp_info.defines) + self.compile_definitions = "\n\t\t\t".join(cpp_info.defines) - self.cppflags = " ".join(deps_cpp_info.cppflags) - self.cflags = " ".join(deps_cpp_info.cflags) - self.sharedlinkflags = " ".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags = " ".join(deps_cpp_info.exelinkflags) + self.cppflags = " ".join(cpp_info.cppflags) + self.cflags = " ".join(cpp_info.cflags) + self.sharedlinkflags = " ".join(cpp_info.sharedlinkflags) + self.exelinkflags = " ".join(cpp_info.exelinkflags) # For modern CMake targets we need to prepare a list to not # loose the elements in the list by replacing " " with ";". Example "-framework Foundation" # Issue: #1251 - self.cppflags_list = ";".join(deps_cpp_info.cppflags) - self.cflags_list = ";".join(deps_cpp_info.cflags) - self.sharedlinkflags_list = ";".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags_list = ";".join(deps_cpp_info.exelinkflags) + self.cppflags_list = ";".join(cpp_info.cppflags) + self.cflags_list = ";".join(cpp_info.cflags) + self.sharedlinkflags_list = ";".join(cpp_info.sharedlinkflags) + self.exelinkflags_list = ";".join(cpp_info.exelinkflags) - self.rootpath = '"%s"' % deps_cpp_info.rootpath.replace("\\", "/") + self.rootpath = '"%s"' % cpp_info.rootpath.replace("\\", "/") class CMakeGenerator(Generator): diff --git a/conans/client/generators/qbs.py b/conans/client/generators/qbs.py index 1dd9e720ffc..3058288c98f 100644 --- a/conans/client/generators/qbs.py +++ b/conans/client/generators/qbs.py @@ -3,24 +3,24 @@ class DepsCppQbs(object): - def __init__(self, deps_cpp_info): + def __init__(self, cpp_info): delimiter = ",\n " self.include_paths = delimiter.join('"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.include_paths) + for p in cpp_info.include_paths) self.lib_paths = delimiter.join('"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.lib_paths) - self.libs = delimiter.join('"%s"' % l for l in deps_cpp_info.libs) - self.defines = delimiter.join('"%s"' % d for d in deps_cpp_info.defines) + for p in cpp_info.lib_paths) + self.libs = delimiter.join('"%s"' % l for l in cpp_info.libs) + self.defines = delimiter.join('"%s"' % d for d in cpp_info.defines) self.cppflags = delimiter.join('"%s"' % d - for d in deps_cpp_info.cppflags) - self.cflags = delimiter.join('"%s"' % d for d in deps_cpp_info.cflags) + for d in cpp_info.cppflags) + self.cflags = delimiter.join('"%s"' % d for d in cpp_info.cflags) self.sharedlinkflags = delimiter.join('"%s"' % d - for d in deps_cpp_info.sharedlinkflags) + for d in cpp_info.sharedlinkflags) self.sharedlinkflags += delimiter.join('"%s"' % d - for d in deps_cpp_info.exelinkflags) + for d in cpp_info.exelinkflags) self.bin_paths = delimiter.join('"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.bin_paths) - self.rootpath = '%s' % deps_cpp_info.rootpath.replace("\\", "/") + for p in cpp_info.bin_paths) + self.rootpath = '%s' % cpp_info.rootpath.replace("\\", "/") class QbsGenerator(Generator): diff --git a/conans/client/generators/qmake.py b/conans/client/generators/qmake.py index 3800196b05a..f735262f548 100644 --- a/conans/client/generators/qmake.py +++ b/conans/client/generators/qmake.py @@ -3,26 +3,26 @@ class DepsCppQmake(object): - def __init__(self, deps_cpp_info): + def __init__(self, cpp_info): def multiline(field): return " \\\n ".join('"%s"' % p.replace("\\", "/") for p in field) - self.include_paths = multiline(deps_cpp_info.include_paths) + self.include_paths = multiline(cpp_info.include_paths) self.lib_paths = " \\\n ".join('-L"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.lib_paths) - self.bin_paths = multiline(deps_cpp_info.bin_paths) - self.res_paths = multiline(deps_cpp_info.res_paths) - self.build_paths = multiline(deps_cpp_info.build_paths) - - self.libs = " ".join('-l%s' % l for l in deps_cpp_info.libs) - self.defines = " \\\n ".join('"%s"' % d for d in deps_cpp_info.defines) - self.cppflags = " ".join(deps_cpp_info.cppflags) - self.cflags = " ".join(deps_cpp_info.cflags) - self.sharedlinkflags = " ".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags = " ".join(deps_cpp_info.exelinkflags) - - self.rootpath = '%s' % deps_cpp_info.rootpath.replace("\\", "/") + for p in cpp_info.lib_paths) + self.bin_paths = multiline(cpp_info.bin_paths) + self.res_paths = multiline(cpp_info.res_paths) + self.build_paths = multiline(cpp_info.build_paths) + + self.libs = " ".join('-l%s' % l for l in cpp_info.libs) + self.defines = " \\\n ".join('"%s"' % d for d in cpp_info.defines) + self.cppflags = " ".join(cpp_info.cppflags) + self.cflags = " ".join(cpp_info.cflags) + self.sharedlinkflags = " ".join(cpp_info.sharedlinkflags) + self.exelinkflags = " ".join(cpp_info.exelinkflags) + + self.rootpath = '%s' % cpp_info.rootpath.replace("\\", "/") class QmakeGenerator(Generator): diff --git a/conans/client/generators/text.py b/conans/client/generators/text.py index 98a88fb8e1a..3e549b9d0dc 100644 --- a/conans/client/generators/text.py +++ b/conans/client/generators/text.py @@ -10,24 +10,24 @@ class DepsCppTXT(object): - def __init__(self, deps_cpp_info): + def __init__(self, cpp_info): self.include_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.include_paths) + for p in cpp_info.include_paths) self.lib_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.lib_paths) + for p in cpp_info.lib_paths) self.res_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.res_paths) + for p in cpp_info.res_paths) self.build_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.build_paths) - self.libs = "\n".join(deps_cpp_info.libs) - self.defines = "\n".join(deps_cpp_info.defines) - self.cppflags = "\n".join(deps_cpp_info.cppflags) - self.cflags = "\n".join(deps_cpp_info.cflags) - self.sharedlinkflags = "\n".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags = "\n".join(deps_cpp_info.exelinkflags) + for p in cpp_info.build_paths) + self.libs = "\n".join(cpp_info.libs) + self.defines = "\n".join(cpp_info.defines) + self.cppflags = "\n".join(cpp_info.cppflags) + self.cflags = "\n".join(cpp_info.cflags) + self.sharedlinkflags = "\n".join(cpp_info.sharedlinkflags) + self.exelinkflags = "\n".join(cpp_info.exelinkflags) self.bin_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.bin_paths) - self.rootpath = "%s" % deps_cpp_info.rootpath.replace("\\", "/") + for p in cpp_info.bin_paths) + self.rootpath = "%s" % cpp_info.rootpath.replace("\\", "/") class TXTGenerator(Generator): @@ -65,7 +65,7 @@ def _loads_deps_user_info(text): @staticmethod def _loads_cpp_info(text): - pattern = re.compile(r"^\[([a-zA-Z0-9_:-\\.]+)\]([^\[]+)", re.MULTILINE) + pattern = re.compile(r"^\[([a-zA-Z0-9._:-]+)\]([^\[]+)", re.MULTILINE) result = DepsCppInfo() try: diff --git a/conans/client/generators/virtualenv.py b/conans/client/generators/virtualenv.py index 14486c5bad5..35cb37378a0 100644 --- a/conans/client/generators/virtualenv.py +++ b/conans/client/generators/virtualenv.py @@ -1,6 +1,5 @@ import os import platform - from conans.model import Generator diff --git a/conans/client/generators/visualstudio.py b/conans/client/generators/visualstudio.py index 35eb7ff3ae1..b92b0a77400 100644 --- a/conans/client/generators/visualstudio.py +++ b/conans/client/generators/visualstudio.py @@ -29,6 +29,12 @@ class VisualStudioGenerator(Generator): {libs}%(AdditionalDependencies) {linker_flags} %(AdditionalOptions) + + {include_dirs}%(AdditionalIncludeDirectories) + + + {include_dirs}%(AdditionalIncludeDirectories) + ''' @@ -38,9 +44,9 @@ class VisualStudioGenerator(Generator): def _format_items(self): sections = [] - for dep_name, dep_cpp_info in self.deps_build_info.dependencies: + for dep_name, cpp_info in self.deps_build_info.dependencies: fields = { - 'root_dir': dep_cpp_info.rootpath, + 'root_dir': cpp_info.rootpath, 'name': dep_name.replace(".", "-") } section = self.item_template.format(**fields) diff --git a/conans/client/generators/visualstudiolegacy.py b/conans/client/generators/visualstudiolegacy.py new file mode 100644 index 00000000000..77e2539e1b7 --- /dev/null +++ b/conans/client/generators/visualstudiolegacy.py @@ -0,0 +1,40 @@ +from conans.model import Generator + + +class VisualStudioLegacyGenerator(Generator): + template = ''' + + + +''' + + @property + def filename(self): + return 'conanbuildinfo.vsprops' + + @property + def content(self): + fields = { + 'include_dirs': "".join(""%s";" % p for p in self._deps_build_info.include_paths).replace("\\", "/"), + 'lib_dirs': "".join(""%s";" % p for p in self._deps_build_info.lib_paths).replace("\\", "/"), + 'libs': "".join(['%s.lib ' % lib if not lib.endswith(".lib") + else '%s ' % lib for lib in self._deps_build_info.libs]), + 'definitions': "".join("%s;" % d for d in self._deps_build_info.defines), + 'compiler_flags': " ".join(self._deps_build_info.cppflags + self._deps_build_info.cflags), + 'linker_flags': " ".join(self._deps_build_info.sharedlinkflags), + } + return self.template.format(**fields) diff --git a/conans/client/importer.py b/conans/client/importer.py index bf6767a44f4..78b39f3ff30 100644 --- a/conans/client/importer.py +++ b/conans/client/importer.py @@ -112,6 +112,6 @@ def _get_folders(self, pattern): each dependency """ if not pattern: - return {pkg: deps.rootpath for pkg, deps in self._conanfile.deps_cpp_info.dependencies} - return {pkg: deps.rootpath for pkg, deps in self._conanfile.deps_cpp_info.dependencies + return {pkg: cpp_info.rootpath for pkg, cpp_info in self._conanfile.deps_cpp_info.dependencies} + return {pkg: cpp_info.rootpath for pkg, cpp_info in self._conanfile.deps_cpp_info.dependencies if fnmatch.fnmatch(pkg, pattern)} diff --git a/conans/client/installer.py b/conans/client/installer.py index 6bc4517d7e2..26acc3302d5 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -1,7 +1,6 @@ import os import time import platform -import fnmatch import shutil from conans.model.env_info import EnvInfo @@ -10,7 +9,8 @@ from conans.util.files import save, rmdir, mkdir from conans.model.ref import PackageReference from conans.util.log import logger -from conans.errors import ConanException, conanfile_exception_formatter, ConanExceptionInUserConanfileMethod +from conans.errors import (ConanException, conanfile_exception_formatter, + ConanExceptionInUserConanfileMethod) from conans.client.packager import create_package from conans.client.generators import write_generators, TXTGenerator from conans.model.build_info import CppInfo @@ -36,76 +36,214 @@ def _init_package_info(deps_graph, paths, current_path): conan_file.user_info = UserInfo() -def build_id(conanfile): - if hasattr(conanfile, "build_id"): +def build_id(conan_file): + if hasattr(conan_file, "build_id"): # construct new ConanInfo - build_id_info = conanfile.info.copy() - conanfile.info_build = build_id_info + build_id_info = conan_file.info.copy() + conan_file.info_build = build_id_info # effectively call the user function to change the package values - with conanfile_exception_formatter(str(conanfile), "build_id"): - conanfile.build_id() + with conanfile_exception_formatter(str(conan_file), "build_id"): + conan_file.build_id() # compute modified ID return build_id_info.package_id() return None -class BuildMode(object): - def __init__(self, params, output): +class _ConanPackageBuilder(object): + """Builds and packages a single conan_file binary package""" + + def __init__(self, conan_file, package_reference, client_cache, output): + self._client_cache = client_cache + self._conan_file = conan_file self._out = output - self.outdated = False - self.missing = False - self.patterns = [] - self._unused_patterns = [] - self.all = False - if params is None: - return - - assert isinstance(params, list) - if len(params) == 0: - self.all = True + self._package_reference = package_reference + self._conan_ref = self._package_reference.conan + self._build_folder = None + + def build(self): + """Calls the conanfile's build method""" + if not os.path.exists(self.build_folder) or not hasattr(self._conan_file, "build_id"): + # build_id is not caching the build folder, so actually rebuild the package + _handle_system_requirements(self._conan_file, self._package_reference, + self._client_cache, self._out) + with environment_append(self._conan_file.env): + self._build_package() + + def package(self): + """Generate the info txt files and calls the conanfile package method. + Receives que build_folder because it can change if build_id() method exists""" + + # FIXME: Is weak to assign here the recipe_hash + manifest = self._client_cache.load_manifest(self._conan_ref) + self._conan_file.info.recipe_hash = manifest.summary_hash + + # Creating ***info.txt files + save(os.path.join(self.build_folder, CONANINFO), self._conan_file.info.dumps()) + self._out.info("Generated %s" % CONANINFO) + save(os.path.join(self.build_folder, BUILD_INFO), TXTGenerator(self._conan_file).content) + self._out.info("Generated %s" % BUILD_INFO) + + os.chdir(self.build_folder) + + if getattr(self._conan_file, 'no_copy_source', False): + source_folder = self._client_cache.source(self._conan_ref, + self._conan_file.short_paths) else: - never = False - for param in params: - if param == "outdated": - self.outdated = True - elif param == "missing": - self.missing = True - elif param == "never": - never = True - else: - self.patterns.append("%s" % param) - - if never and (self.outdated or self.missing or self.patterns): - raise ConanException("--build=never not compatible with other options") - self._unused_patterns = list(self.patterns) - - def forced(self, reference, conanfile): - if self.all: - return True + source_folder = self.build_folder + with environment_append(self._conan_file.env): + package_folder = self._client_cache.package(self._package_reference, + self._conan_file.short_paths) + create_package(self._conan_file, source_folder, self.build_folder, package_folder, + self._out) + + def _build_package(self): + """ builds the package, creating the corresponding build folder if necessary + and copying there the contents from the src folder. The code is duplicated + in every build, as some configure processes actually change the source + code. Receives the build_folder because it can change if the method build_id() exists + """ - if conanfile.build_policy_always: - out = ScopedOutput(str(reference), self._out) - out.info("Building package from source as defined by build_policy='always'") - return True + package_folder = self._client_cache.package(self._package_reference, + self._conan_file.short_paths) + src_folder = self._client_cache.source(self._conan_ref, self._conan_file.short_paths) + export_folder = self._client_cache.export(self._conan_ref) + export_source_folder = self._client_cache.export_sources(self._conan_ref, + self._conan_file.short_paths) + + try: + rmdir(self.build_folder) + rmdir(package_folder) + except Exception as e: + raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n" + "Close any app using it, and retry" % str(e)) - ref = reference.name - # Patterns to match, if package matches pattern, build is forced - force_build = any([fnmatch.fnmatch(ref, pattern) for pattern in self.patterns]) - return force_build + self._out.info('Building your package in %s' % self.build_folder) + config_source(export_folder, export_source_folder, src_folder, + self._conan_file, self._out) + self._out.info('Copying sources to build folder') - def allowed(self, reference, conanfile): - return (self.missing or self.outdated or self.forced(reference, conanfile) or - conanfile.build_policy_missing) + if getattr(self._conan_file, 'no_copy_source', False): + mkdir(self.build_folder) + self._conan_file.source_folder = src_folder + else: + if platform.system() == "Windows" and os.getenv("CONAN_USER_HOME_SHORT") != "None": + from conans.util.windows import ignore_long_path_files + ignore = ignore_long_path_files(src_folder, self.build_folder, self._out) + else: + ignore = None - def check_matches(self, references): - for pattern in list(self._unused_patterns): - matched = any(fnmatch.fnmatch(ref, pattern) for ref in references) - if matched: - self._unused_patterns.remove(pattern) + shutil.copytree(src_folder, self.build_folder, symlinks=True, ignore=ignore) + logger.debug("Copied to %s" % self.build_folder) + logger.debug("Files copied %s" % os.listdir(self.build_folder)) + self._conan_file.source_folder = self.build_folder - def report_matches(self): - for pattern in self._unused_patterns: - self._out.error("No package matching '%s' pattern" % pattern) + os.chdir(self.build_folder) + self._conan_file.build_folder = self.build_folder + self._conan_file._conanfile_directory = self.build_folder + # Read generators from conanfile and generate the needed files + logger.debug("Writing generators") + write_generators(self._conan_file, self.build_folder, self._out) + logger.debug("Files copied after generators %s" % os.listdir(self.build_folder)) + + # Build step might need DLLs, binaries as protoc to generate source files + # So execute imports() before build, storing the list of copied_files + from conans.client.importer import run_imports + copied_files = run_imports(self._conan_file, self.build_folder, self._out) + + try: + # This is necessary because it is different for user projects + # than for packages + logger.debug("Call conanfile.build() with files in build folder: %s" + % os.listdir(self.build_folder)) + self._out.highlight("Calling build()") + with conanfile_exception_formatter(str(self._conan_file), "build"): + self._conan_file.build() + + self._out.success("Package '%s' built" % self._conan_file.info.package_id()) + self._out.info("Build folder %s" % self.build_folder) + except Exception as exc: + os.chdir(src_folder) + self._out.writeln("") + self._out.error("Package '%s' build failed" % self._conan_file.info.package_id()) + self._out.warn("Build folder %s" % self.build_folder) + if isinstance(exc, ConanExceptionInUserConanfileMethod): + raise exc + raise ConanException(exc) + + finally: + self._conan_file._conanfile_directory = export_folder + # Now remove all files that were imported with imports() + for f in copied_files: + try: + if f.startswith(self.build_folder): + os.remove(f) + except Exception: + self._out.warn("Unable to remove imported file from build: %s" % f) + + @property + def build_folder(self): + if not self._build_folder: + new_id = build_id(self._conan_file) + new_ref = PackageReference(self._conan_ref, new_id) if new_id else self._package_reference + self._build_folder = self._client_cache.build(new_ref, self._conan_file.short_paths) + return self._build_folder + + +def _raise_package_not_found_error(conan_file, conan_ref, out): + settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines()) + options_text = ", ".join(conan_file.info.full_options.dumps().splitlines()) + + out.warn('''Can't find a '%s' package for the specified options and settings: +- Settings: %s +- Options: %s +''' % (conan_ref, settings_text, options_text)) + + raise ConanException('''Missing prebuilt package for '%s' +Try to build it from sources with "--build %s" +Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package" +''' % (conan_ref, conan_ref.name)) + + +def _handle_system_requirements(conan_file, package_reference, client_cache, out): + """ check first the system_reqs/system_requirements.txt existence, if not existing + check package/sha1/ + + Used after remote package retrieving and before package building + """ + if "system_requirements" not in type(conan_file).__dict__: + return + + system_reqs_path = client_cache.system_reqs(package_reference.conan) + system_reqs_package_path = client_cache.system_reqs_package(package_reference) + if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path): + return + + ret = call_system_requirements(conan_file, out) + + try: + ret = str(ret or "") + except: + out.warn("System requirements didn't return a string") + ret = "" + if getattr(conan_file, "global_system_requirements", None): + save(system_reqs_path, ret) + else: + save(system_reqs_package_path, ret) + + +def call_system_requirements(conanfile, output): + try: + return conanfile.system_requirements() + except Exception as e: + output.error("while executing system_requirements(): %s" % str(e)) + raise ConanException("Error in system requirements") + + +def call_package_info(conanfile): + # Once the node is build, execute package info, so it has access to the + # package folder and artifacts + with conanfile_exception_formatter(str(conanfile), "package_info"): + conanfile.package_info() class ConanInstaller(object): @@ -149,7 +287,7 @@ def _compute_private_nodes(self, deps_graph): continue if conan_ref: - build_forced = self._build_mode.forced(conan_ref, conanfile) + build_forced = self._build_mode.forced(conanfile, conan_ref) if build_forced: continue @@ -200,55 +338,79 @@ def _build(self, nodes_by_level, skip_private_nodes, deps_graph): nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes) for conan_ref, package_id, conan_file, build_needed in nodes_to_process: + output = ScopedOutput(str(conan_ref), self._out) + package_ref = PackageReference(conan_ref, package_id) + if build_needed and (conan_ref, package_id) not in self._built_packages: - build_allowed = self._build_mode.allowed(conan_ref, conan_file) + build_allowed = self._build_mode.allowed(conan_file, conan_ref) if not build_allowed: - self._raise_package_not_found_error(conan_ref, conan_file) + _raise_package_not_found_error(conan_file, conan_ref, output) - output = ScopedOutput(str(conan_ref), self._out) - package_ref = PackageReference(conan_ref, package_id) - package_folder = self._client_cache.package(package_ref, conan_file.short_paths) if conan_file.build_policy_missing: output.info("Building package from source as defined by build_policy='missing'") - elif self._build_mode.forced(conan_ref, conan_file): + elif self._build_mode.forced(conan_file, conan_ref): output.warn('Forced build from source') self._build_requires.install(conan_ref, conan_file, self) t1 = time.time() # Assign to node the propagated info - self._propagate_info(conan_ref, conan_file, flat, deps_graph) + self._propagate_info(conan_file, conan_ref, flat, deps_graph) self._remote_proxy.get_recipe_sources(conan_ref, conan_file.short_paths) - # Call the conanfile's build method - build_folder = self._build_conanfile(conan_ref, conan_file, package_ref, - package_folder, output) + builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output) + builder.build() + builder.package() - # Call the conanfile's package method - self._package_conanfile(conan_ref, conan_file, package_ref, build_folder, - package_folder, output) + self._remote_proxy.handle_package_manifest(package_ref, installed=True) # Call the info method - self._package_info_conanfile(conan_file) + call_package_info(conan_file) - duration = time.time() - t1 - log_file = os.path.join(build_folder, RUN_LOG_NAME) - log_file = log_file if os.path.exists(log_file) else None - log_package_built(package_ref, duration, log_file) + # Log build + self._log_built_package(conan_file, package_ref, time.time() - t1) self._built_packages.add((conan_ref, package_id)) else: # Get the package, we have a not outdated remote package if conan_ref: - self._get_package(conan_ref, conan_file) + self.get_remote_package(conan_file, package_ref, output) # Assign to the node the propagated info # (conan_ref could be None if user project, but of course assign the info - self._propagate_info(conan_ref, conan_file, flat, deps_graph) + self._propagate_info(conan_file, conan_ref, flat, deps_graph) # Call the info method - self._package_info_conanfile(conan_file) + call_package_info(conan_file) + + def get_remote_package(self, conan_file, package_reference, output): + """Get remote package. It won't check if it's outdated""" + # Compute conan_file package from local (already compiled) or from remote + + package_folder = self._client_cache.package(package_reference, + conan_file.short_paths) + + # If already exists do not dirt the output, the common situation + # is that package is already installed and OK. If don't, the proxy + # will print some other message about it + if not os.path.exists(package_folder): + self._out.info("Retrieving package %s" % package_reference.package_id) - def _propagate_info(self, conan_ref, conan_file, flat, deps_graph): + if self._remote_proxy.get_package(package_reference, + short_paths=conan_file.short_paths): + _handle_system_requirements(conan_file, package_reference, + self._client_cache, output) + return True + + _raise_package_not_found_error(conan_file, package_reference.conan, output) + + def _log_built_package(self, conan_file, package_ref, duration): + build_folder = self._client_cache.build(package_ref, conan_file.short_paths) + log_file = os.path.join(build_folder, RUN_LOG_NAME) + log_file = log_file if os.path.exists(log_file) else None + log_package_built(package_ref, duration, log_file) + + @staticmethod + def _propagate_info(conan_file, conan_ref, flat, deps_graph): # Get deps_cpp_info from upstream nodes node_order = deps_graph.ordered_closure((conan_ref, conan_file), flat) public_deps = [name for name, req in conan_file.requires.items() if not req.private] @@ -266,7 +428,8 @@ def _propagate_info(self, conan_ref, conan_file, flat, deps_graph): subtree_libnames = [ref.name for (ref, _) in node_order] for package_name, env_vars in conan_file._env_values.data.items(): for name, value in env_vars.items(): - if not package_name or package_name in subtree_libnames or package_name == conan_file.name: + if not package_name or package_name in subtree_libnames or \ + package_name == conan_file.name: conan_file.info.env_values.add(name, value, package_name) def _get_nodes(self, nodes_by_level, skip_nodes): @@ -296,12 +459,13 @@ def _get_nodes(self, nodes_by_level, skip_nodes): if package_reference not in package_references: package_references.add(package_reference) check_outdated = self._build_mode.outdated - if self._build_mode.forced(conan_ref, conan_file): + if self._build_mode.forced(conan_file, conan_ref): build_node = True else: - build_node = not self._remote_proxy.package_available(package_reference, - conan_file.short_paths, - check_outdated) + available = self._remote_proxy.package_available(package_reference, + conan_file.short_paths, + check_outdated) + build_node = not available nodes_to_build.append((conan_ref, package_id, conan_file, build_node)) @@ -312,206 +476,3 @@ def _get_nodes(self, nodes_by_level, skip_nodes): self._build_mode.check_matches(to_build) return nodes_to_build - - def _get_package(self, conan_ref, conan_file): - '''Get remote package. It won't check if it's outdated''' - # Compute conan_file package from local (already compiled) or from remote - output = ScopedOutput(str(conan_ref), self._out) - package_id = conan_file.info.package_id() - package_reference = PackageReference(conan_ref, package_id) - - conan_ref = package_reference.conan - package_folder = self._client_cache.package(package_reference, conan_file.short_paths) - - # If already exists do not dirt the output, the common situation - # is that package is already installed and OK. If don't, the proxy - # will print some other message about it - if not os.path.exists(package_folder): - output.info("Retrieving package %s" % package_id) - - if self._remote_proxy.get_package(package_reference, short_paths=conan_file.short_paths): - self._handle_system_requirements(conan_ref, package_reference, conan_file, output) - return True - - self._raise_package_not_found_error(conan_ref, conan_file) - - def _build_conanfile(self, conan_ref, conan_file, package_reference, package_folder, output): - """Calls the conanfile's build method""" - new_id = build_id(conan_file) - if new_id: - package_reference = PackageReference(package_reference.conan, new_id) - build_folder = self._client_cache.build(package_reference, conan_file.short_paths) - if os.path.exists(build_folder) and hasattr(conan_file, "build_id"): - return build_folder - # build_id is not caching the build folder, so actually rebuild the package - src_folder = self._client_cache.source(conan_ref, conan_file.short_paths) - export_folder = self._client_cache.export(conan_ref) - export_source_folder = self._client_cache.export_sources(conan_ref, conan_file.short_paths) - - self._handle_system_requirements(conan_ref, package_reference, conan_file, output) - with environment_append(conan_file.env): - self._build_package(export_folder, export_source_folder, src_folder, build_folder, - package_folder, conan_file, output) - return build_folder - - def _package_conanfile(self, conan_ref, conan_file, package_reference, build_folder, - package_folder, output): - """Generate the info txt files and calls the conanfile package method""" - - # FIXME: Is weak to assign here the recipe_hash - conan_file.info.recipe_hash = self._client_cache.load_manifest(conan_ref).summary_hash - - # Creating ***info.txt files - save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) - output.info("Generated %s" % CONANINFO) - save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content) - output.info("Generated %s" % BUILD_INFO) - - os.chdir(build_folder) - - if getattr(conan_file, 'no_copy_source', False): - source_folder = self._client_cache.source(package_reference.conan, - conan_file.short_paths) - else: - source_folder = build_folder - with environment_append(conan_file.env): - create_package(conan_file, source_folder, build_folder, package_folder, output) - self._remote_proxy.handle_package_manifest(package_reference, installed=True) - - def _raise_package_not_found_error(self, conan_ref, conan_file): - settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines()) - options_text = ", ".join(conan_file.info.full_options.dumps().splitlines()) - - self._out.warn('''Can't find a '%s' package for the specified options and settings: -- Settings: %s -- Options: %s -''' % (conan_ref, settings_text, options_text)) - - raise ConanException('''Missing prebuilt package for '%s' -Try to build it from sources with "--build %s" -Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package" -''' % (conan_ref, conan_ref.name)) - - def _handle_system_requirements(self, conan_ref, package_reference, conan_file, coutput): - """ check first the system_reqs/system_requirements.txt existence, if not existing - check package/sha1/ - """ - if "system_requirements" not in type(conan_file).__dict__: - return - - system_reqs_path = self._client_cache.system_reqs(conan_ref) - system_reqs_package_path = self._client_cache.system_reqs_package(package_reference) - if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path): - return - - output = self.call_system_requirements(conan_file, coutput) - - try: - output = str(output or "") - except: - coutput.warn("System requirements didn't return a string") - output = "" - if getattr(conan_file, "global_system_requirements", None): - save(system_reqs_path, output) - else: - save(system_reqs_package_path, output) - - def call_system_requirements(self, conan_file, output): - try: - return conan_file.system_requirements() - except Exception as e: - output.error("while executing system_requirements(): %s" % str(e)) - raise ConanException("Error in system requirements") - - def _build_package(self, export_folder, export_source_folder, src_folder, build_folder, - package_folder, conan_file, output): - """ builds the package, creating the corresponding build folder if necessary - and copying there the contents from the src folder. The code is duplicated - in every build, as some configure processes actually change the source - code - """ - - try: - rmdir(build_folder) - rmdir(package_folder) - except Exception as e: - raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n" - "Close any app using it, and retry" % str(e)) - - output.info('Building your package in %s' % build_folder) - config_source(export_folder, export_source_folder, src_folder, conan_file, output) - output.info('Copying sources to build folder') - - def check_max_path_len(src, files): - if platform.system() != "Windows": - return [] - filtered_files = [] - for the_file in files: - source_path = os.path.join(src, the_file) - # Without storage path, just relative - rel_path = os.path.relpath(source_path, src_folder) - dest_path = os.path.normpath(os.path.join(build_folder, rel_path)) - # it is NOT that "/" is counted as "\\" so it counts double - # seems a bug in python, overflows paths near the limit of 260, - if len(dest_path) >= 249: - filtered_files.append(the_file) - output.warn("Filename too long, file excluded: %s" % dest_path) - return filtered_files - - if getattr(conan_file, 'no_copy_source', False): - mkdir(build_folder) - conan_file.source_folder = src_folder - else: - shutil.copytree(src_folder, build_folder, symlinks=True, ignore=check_max_path_len) - logger.debug("Copied to %s" % build_folder) - logger.debug("Files copied %s" % os.listdir(build_folder)) - conan_file.source_folder = build_folder - - os.chdir(build_folder) - conan_file.build_folder = build_folder - conan_file._conanfile_directory = build_folder - # Read generators from conanfile and generate the needed files - logger.debug("Writing generators") - write_generators(conan_file, build_folder, output) - logger.debug("Files copied after generators %s" % os.listdir(build_folder)) - - # Build step might need DLLs, binaries as protoc to generate source files - # So execute imports() before build, storing the list of copied_files - from conans.client.importer import run_imports - copied_files = run_imports(conan_file, build_folder, output) - - try: - # This is necessary because it is different for user projects - # than for packages - logger.debug("Call conanfile.build() with files in build folder: %s" - % os.listdir(build_folder)) - output.highlight("Calling build()") - with conanfile_exception_formatter(str(conan_file), "build"): - conan_file.build() - - output.success("Package '%s' built" % conan_file.info.package_id()) - output.info("Build folder %s" % build_folder) - except Exception as exc: - os.chdir(src_folder) - self._out.writeln("") - output.error("Package '%s' build failed" % conan_file.info.package_id()) - output.warn("Build folder %s" % build_folder) - if isinstance(exc, ConanExceptionInUserConanfileMethod): - raise exc - raise ConanException(exc) - - finally: - conan_file._conanfile_directory = export_folder - # Now remove all files that were imported with imports() - for f in copied_files: - try: - if(f.startswith(build_folder)): - os.remove(f) - except Exception: - self._out.warn("Unable to remove imported file from build: %s" % f) - - def _package_info_conanfile(self, conan_file): - # Once the node is build, execute package info, so it has access to the - # package folder and artifacts - with conanfile_exception_formatter(str(conan_file), "package_info"): - conan_file.package_info() diff --git a/conans/client/loader.py b/conans/client/loader.py index c31946cd766..bb20887aeb2 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -66,6 +66,7 @@ def load_conan(self, conanfile_path, output, consumer=False, reference=None): result.scope = self._scopes.package_scope() else: result.scope = self._scopes.package_scope(result.name) + result.in_local_cache = True return result except Exception as e: # re-raise with file name diff --git a/conans/client/loader_parse.py b/conans/client/loader_parse.py index e810b642901..457215e63fc 100644 --- a/conans/client/loader_parse.py +++ b/conans/client/loader_parse.py @@ -4,13 +4,13 @@ import sys import uuid -from conans.client.generators import _save_generator from conans.errors import ConanException, NotFoundException from conans.model.conan_file import ConanFile -from conans.model.conan_generator import Generator from conans.util.config_parser import ConfigParser from conans.util.files import rmdir from conans.tools import chdir +from conans.client.generators import registered_generators +from conans.model import Generator def load_conanfile_class(conanfile_path): @@ -40,7 +40,7 @@ class defining the Recipe, but also process possible existing generators raise ConanException("More than 1 conanfile in the file") if (inspect.isclass(attr) and issubclass(attr, Generator) and attr != Generator and attr.__dict__["__module__"] == filename): - _save_generator(attr.__name__, attr) + registered_generators.add(attr.__name__, attr) if result is None: raise ConanException("No subclass of ConanFile") diff --git a/conans/client/manager.py b/conans/client/manager.py index 357a6459c42..3adde12c21c 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -1,3 +1,4 @@ +import fnmatch import os import time import shutil @@ -13,7 +14,7 @@ from conans.client.generators import write_generators from conans.client.generators.text import TXTGenerator from conans.client.importer import run_imports, undo_imports -from conans.client.installer import ConanInstaller, BuildMode +from conans.client.installer import ConanInstaller, call_system_requirements from conans.client.loader import ConanFileLoader from conans.client.manifest_manager import ManifestManager from conans.client.output import ScopedOutput, Color @@ -40,6 +41,65 @@ from conans.search.search import filter_outdated +class BuildMode(object): + def __init__(self, params, output): + self._out = output + self.outdated = False + self.missing = False + self.patterns = [] + self._unused_patterns = [] + self.all = False + if params is None: + return + + assert isinstance(params, list) + if len(params) == 0: + self.all = True + else: + never = False + for param in params: + if param == "outdated": + self.outdated = True + elif param == "missing": + self.missing = True + elif param == "never": + never = True + else: + self.patterns.append("%s" % param) + + if never and (self.outdated or self.missing or self.patterns): + raise ConanException("--build=never not compatible with other options") + self._unused_patterns = list(self.patterns) + + def forced(self, conan_file, reference): + if self.all: + return True + + if conan_file.build_policy_always: + out = ScopedOutput(str(reference), self._out) + out.info("Building package from source as defined by build_policy='always'") + return True + + ref = reference.name + # Patterns to match, if package matches pattern, build is forced + force_build = any([fnmatch.fnmatch(ref, pattern) for pattern in self.patterns]) + return force_build + + def allowed(self, conan_file, reference): + return (self.missing or self.outdated or self.forced(conan_file, reference) or + conan_file.build_policy_missing) + + def check_matches(self, references): + for pattern in list(self._unused_patterns): + matched = any(fnmatch.fnmatch(ref, pattern) for ref in references) + if matched: + self._unused_patterns.remove(pattern) + + def report_matches(self): + for pattern in self._unused_patterns: + self._out.error("No package matching '%s' pattern" % pattern) + + class ConanManager(object): """ Manage all the commands logic The main entry point for all the client business logic @@ -359,7 +419,7 @@ def install(self, reference, current_path, profile, remote=None, output.info("Generated %s" % CONANINFO) if not no_imports: run_imports(conanfile, current_path, output) - installer.call_system_requirements(conanfile, output) + call_system_requirements(conanfile, output) if manifest_manager: manifest_manager.print_log() diff --git a/conans/client/manifest_manager.py b/conans/client/manifest_manager.py index ce566375a25..40899b74f14 100644 --- a/conans/client/manifest_manager.py +++ b/conans/client/manifest_manager.py @@ -18,7 +18,7 @@ def __init__(self, folder, user_io, client_cache, verify=False, interactive=Fals self._log = [] def print_log(self): - self._user_io.out.success("\nManifests") + self._user_io.out.success("\nManifests : %s" % (self._paths.store)) for log_entry in self._log: self._user_io.out.info(log_entry) diff --git a/conans/client/migrations.py b/conans/client/migrations.py index bec3d432c57..cbb23278a20 100644 --- a/conans/client/migrations.py +++ b/conans/client/migrations.py @@ -4,6 +4,7 @@ from conans.client.client_cache import CONAN_CONF, PROFILES_FOLDER from conans.errors import ConanException from conans.migrations import Migrator, CONAN_VERSION +from conans.paths import EXPORT_SOURCES_DIR_OLD from conans.util.files import load, save from conans.model.version import Version @@ -131,7 +132,7 @@ def migrate_c_src_export_source(client_cache, out): package_folders = list_folder_subdirs(client_cache.store, 4) for package in package_folders: package_folder = os.path.join(client_cache.store, package) - c_src = os.path.join(package_folder, "export/.c_src") + c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD) if os.path.exists(c_src): out.warn("Migration: Removing package with old export_sources layout: %s" % package) try: diff --git a/conans/client/new_ci.py b/conans/client/new_ci.py index 0fba66bf964..2dc8c80c474 100644 --- a/conans/client/new_ci.py +++ b/conans/client/new_ci.py @@ -190,6 +190,7 @@ def get_travis(name, version, user, channel, linux_gcc_versions, linux_clang_ver ".travis/run.sh": travis_run} return files + def get_appveyor(name, version, user, channel, visual_versions, upload_url): config = [] visual_config = """ - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio {image} @@ -205,6 +206,7 @@ def get_appveyor(name, version, user, channel, visual_versions, upload_url): channel=channel, configs=configs, upload=upload)} return files + def get_gitlab(name, version, user, channel, linux_gcc_versions, linux_clang_versions, upload_url): config = [] @@ -219,13 +221,14 @@ def get_gitlab(name, version, user, channel, linux_gcc_versions, linux_clang_ver configs = "".join(config) upload = ('CONAN_UPLOAD: "%s"\n' % upload_url) if upload_url else "" files = {".gitlab-ci.yml": gitlab.format(name=name, version=version, user=user, channel=channel, - configs=configs, upload=upload)} + configs=configs, upload=upload)} return files + def ci_get_files(name, version, user, channel, visual_versions, linux_gcc_versions, linux_clang_versions, osx_clang_versions, shared, upload_url, gitlab_gcc_versions, gitlab_clang_versions): if shared and not (visual_versions or linux_gcc_versions or linux_clang_versions or osx_clang_versions or - gitlab_gcc_versions or gitlab_clang_versions): + gitlab_gcc_versions or gitlab_clang_versions): raise ConanException("Trying to specify 'shared' in CI, but no CI system specified") if not (visual_versions or linux_gcc_versions or linux_clang_versions or osx_clang_versions or gitlab_gcc_versions or gitlab_clang_versions): diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index b1155026384..4927d1b51be 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -8,8 +8,8 @@ from conans.errors import ConanException, ConanConnectionError, NotFoundException from conans.model.manifest import gather_files -from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME,\ - rm_conandir, EXPORT_SOURCES_TGZ_NAME +from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME, \ + rm_conandir, EXPORT_SOURCES_TGZ_NAME, EXPORT_SOURCES_DIR_OLD from conans.util.files import gzopen_without_timestamps from conans.util.files import tar_extract, rmdir, exception_message_safe, mkdir from conans.util.files import touch @@ -193,7 +193,7 @@ def filter_function(urls): return unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME) - c_src_path = os.path.join(export_sources_folder, ".c_src") + c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD) if os.path.exists(c_src_path): merge_directories(c_src_path, export_sources_folder) rmdir(c_src_path) @@ -301,7 +301,6 @@ def compress_files(files, symlinks, name, dest_dir): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) - is_export_sources = (name == EXPORT_SOURCES_TGZ_NAME) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) @@ -313,8 +312,6 @@ def compress_files(files, symlinks, name, dest_dir): tgz.addfile(tarinfo=info) for filename, abs_path in files.items(): - if is_export_sources: # temporary backwards compat TGZ creation - filename = ".c_src/%s" % filename info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode diff --git a/conans/client/remote_registry.py b/conans/client/remote_registry.py index 534d5576761..b6ebed6943d 100644 --- a/conans/client/remote_registry.py +++ b/conans/client/remote_registry.py @@ -173,6 +173,15 @@ def exists_function(remotes): raise ConanException("Remote '%s' not found in remotes" % remote_name) self._add_update(remote_name, remote, verify_ssl, exists_function, insert) + def define_remotes(self, remotes): + with fasteners.InterProcessLock(self._filename + ".lock", logger=logger): + _, refs = self._load() + new_remotes = OrderedDict() + for remote in remotes: + new_remotes[remote.name] = (remote.url, remote.verify_ssl) + refs = {k: v for k, v in refs.items() if v in new_remotes} + self._save(new_remotes, refs) + def _add_update(self, remote_name, remote, verify_ssl, exists_function, insert=None): with fasteners.InterProcessLock(self._filename + ".lock", logger=logger): diff --git a/conans/client/require_resolver.py b/conans/client/require_resolver.py index fa0896b722d..d0cf3900ad0 100644 --- a/conans/client/require_resolver.py +++ b/conans/client/require_resolver.py @@ -1,6 +1,5 @@ from conans.model.ref import ConanFileReference from conans.errors import ConanException -import re def satisfying(list_versions, versionexpr, output): @@ -14,8 +13,6 @@ def satisfying(list_versions, versionexpr, output): for v in list_versions: try: ver = SemVer(v, loose=True) - if not ver.prerelease: # Hack to allow version "2.1" match expr "<=2.1" - ver.prerelease = [0] candidates[ver] = v except (ValueError, AttributeError): output.warn("Version '%s' is not semver, cannot be compared with a range" % str(v)) @@ -24,7 +21,6 @@ def satisfying(list_versions, versionexpr, output): class RequireResolver(object): - expr_pattern = re.compile("") def __init__(self, output, local_search, remote_search): self._output = output diff --git a/conans/client/source.py b/conans/client/source.py index 3b0aa7a0c31..b8ef83a2241 100644 --- a/conans/client/source.py +++ b/conans/client/source.py @@ -6,8 +6,8 @@ from conans import tools from conans.errors import ConanException, conanfile_exception_formatter, \ ConanExceptionInUserConanfileMethod -from conans.paths import DIRTY_FILE, EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE -from conans.util.files import rmdir, save +from conans.paths import EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE, CONAN_MANIFEST +from conans.util.files import rmdir, set_dirty, is_dirty, clean_dirty def merge_directories(src, dst): @@ -21,18 +21,18 @@ def merge_directories(src, dst): shutil.copy2(src_file, dst_file) -def config_source(export_folder, export_source_folder, src_folder, conan_file, output, force=False): +def config_source(export_folder, export_source_folder, src_folder, + conan_file, output, force=False): """ creates src folder and retrieve, calling source() from conanfile the necessary source code """ - dirty = os.path.join(src_folder, DIRTY_FILE) def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: - save(dirty, "") # Creation of DIRTY flag + set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1 @@ -44,7 +44,7 @@ def remove_source(raise_error=True): if force: output.warn("Forced removal of source folder") remove_source() - elif os.path.exists(dirty): + elif is_dirty(src_folder): output.warn("Trying to remove dirty source folder") remove_source() elif conan_file.build_policy_always: @@ -56,7 +56,8 @@ def remove_source(raise_error=True): shutil.copytree(export_folder, src_folder, symlinks=True) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) - for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c", CONANFILE+"o"): + for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c", + CONANFILE+"o", CONANFILE, CONAN_MANIFEST): try: os.remove(os.path.join(src_folder, f)) except OSError: @@ -66,13 +67,13 @@ def remove_source(raise_error=True): except OSError: pass - save(dirty, "") # Creation of DIRTY flag + set_dirty(src_folder) os.chdir(src_folder) try: with tools.environment_append(conan_file.env): with conanfile_exception_formatter(str(conan_file), "source"): conan_file.source() - os.remove(dirty) # Everything went well, remove DIRTY flag + clean_dirty(src_folder) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder @@ -86,17 +87,12 @@ def remove_source(raise_error=True): def config_source_local(current_path, conan_file, output): output.info('Configuring sources in %s' % current_path) - dirty = os.path.join(current_path, DIRTY_FILE) - if os.path.exists(dirty): - output.warn("Your previous source command failed") - - save(dirty, "") # Creation of DIRTY flag - try: - with conanfile_exception_formatter(str(conan_file), "source"): - with tools.environment_append(conan_file.env): - conan_file.source() - os.remove(dirty) # Everything went well, remove DIRTY flag - except ConanExceptionInUserConanfileMethod: - raise - except Exception as e: - raise ConanException(e) + with tools.chdir(current_path): + try: + with conanfile_exception_formatter(str(conan_file), "source"): + with tools.environment_append(conan_file.env): + conan_file.source() + except ConanExceptionInUserConanfileMethod: + raise + except Exception as e: + raise ConanException(e) diff --git a/conans/client/tools/__init__.py b/conans/client/tools/__init__.py new file mode 100644 index 00000000000..a36eae84c0f --- /dev/null +++ b/conans/client/tools/__init__.py @@ -0,0 +1,13 @@ + +# noinspection PyUnresolvedReferences +from .env import * +# noinspection PyUnresolvedReferences +from .files import * +# noinspection PyUnresolvedReferences +from .net import * +# noinspection PyUnresolvedReferences +from .oss import * +# noinspection PyUnresolvedReferences +from .system_pm import * +# noinspection PyUnresolvedReferences +from .win import * diff --git a/conans/client/tools/env.py b/conans/client/tools/env.py new file mode 100644 index 00000000000..2c60854efd9 --- /dev/null +++ b/conans/client/tools/env.py @@ -0,0 +1,40 @@ +import sys +from contextlib import contextmanager + +import os + + +@contextmanager +def pythonpath(conanfile): + old_path = sys.path[:] + python_path = conanfile.env.get("PYTHONPATH", None) + if python_path: + if isinstance(python_path, list): + sys.path.extend(python_path) + else: + sys.path.append(python_path) + + yield + sys.path = old_path + + +@contextmanager +def environment_append(env_vars): + """ + :param env_vars: List of simple environment vars. {name: value, name2: value2} => e.j: MYVAR=1 + The values can also be lists of appendable environment vars. {name: [value, value2]} + => e.j. PATH=/path/1:/path/2 + :return: None + """ + old_env = dict(os.environ) + for name, value in env_vars.items(): + if isinstance(value, list): + env_vars[name] = os.pathsep.join(value) + if name in old_env: + env_vars[name] += os.pathsep + old_env[name] + os.environ.update(env_vars) + try: + yield + finally: + os.environ.clear() + os.environ.update(old_env) diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py new file mode 100644 index 00000000000..9601b899f0b --- /dev/null +++ b/conans/client/tools/files.py @@ -0,0 +1,222 @@ +import platform +from contextlib import contextmanager + +import logging + +import re + +import os +import sys + +from conans.client.output import ConanOutput +from conans.errors import ConanException +from conans.util.files import load, save, _generic_algorithm_sum +from patch import fromfile, fromstring + + +_global_output = None + + +@contextmanager +def chdir(newdir): + old_path = os.getcwd() + os.chdir(newdir) + try: + yield + finally: + os.chdir(old_path) + + +def human_size(size_bytes): + """ + format a size in bytes into a 'human' file size, e.g. B, KB, MB, GB, TB, PB + Note that bytes will be reported in whole numbers but KB and above will have + greater precision. e.g. 43 B, 443 KB, 4.3 MB, 4.43 GB, etc + """ + + suffixes_table = [('B', 0), ('KB', 1), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)] + + num = float(size_bytes) + for suffix, precision in suffixes_table: + if num < 1024.0: + break + num /= 1024.0 + + if precision == 0: + formatted_size = "%d" % num + else: + formatted_size = str(round(num, ndigits=precision)) + + return "%s%s" % (formatted_size, suffix) + + +def unzip(filename, destination=".", keep_permissions=False): + """ + Unzip a zipped file + :param filename: Path to the zip file + :param destination: Destination folder + :param keep_permissions: Keep the zip permissions. WARNING: Can be dangerous if the zip was not created in a NIX + system, the bits could produce undefined permission schema. Use only this option if you are sure that the + zip was created correctly. + :return: + """ + if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or + filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or + filename.endswith(".tar")): + return untargz(filename, destination) + import zipfile + full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) + + if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + def print_progress(the_size, uncomp_size): + the_size = (the_size * 100.0 / uncomp_size) if uncomp_size != 0 else 0 + txt_msg = "Unzipping %.0f %%" % the_size + _global_output.rewrite_line(txt_msg) + else: + def print_progress(_, __): + pass + + with zipfile.ZipFile(filename, "r") as z: + uncompress_size = sum((file_.file_size for file_ in z.infolist())) + if uncompress_size > 100000: + _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) + else: + _global_output.info("Unzipping %s" % human_size(uncompress_size)) + extracted_size = 0 + if platform.system() == "Windows": + for file_ in z.infolist(): + extracted_size += file_.file_size + print_progress(extracted_size, uncompress_size) + try: + # Win path limit is 260 chars + if len(file_.filename) + len(full_path) >= 260: + raise ValueError("Filename too long") + z.extract(file_, full_path) + except Exception as e: + _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) + else: # duplicated for, to avoid a platform check for each zipped file + for file_ in z.infolist(): + extracted_size += file_.file_size + print_progress(extracted_size, uncompress_size) + try: + z.extract(file_, full_path) + if keep_permissions: + # Could be dangerous if the ZIP has been created in a non nix system + # https://bugs.python.org/issue15795 + perm = file_.external_attr >> 16 & 0xFFF + os.chmod(os.path.join(full_path, file_.filename), perm) + except Exception as e: + _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) + + +def untargz(filename, destination="."): + import tarfile + with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile: + tarredgzippedFile.extractall(destination) + + +def check_with_algorithm_sum(algorithm_name, file_path, signature): + real_signature = _generic_algorithm_sum(file_path, algorithm_name) + if real_signature != signature: + raise ConanException("%s signature failed for '%s' file." + " Computed signature: %s" % (algorithm_name, + os.path.basename(file_path), + real_signature)) + + +def check_sha1(file_path, signature): + check_with_algorithm_sum("sha1", file_path, signature) + + +def check_md5(file_path, signature): + check_with_algorithm_sum("md5", file_path, signature) + + +def check_sha256(file_path, signature): + check_with_algorithm_sum("sha256", file_path, signature) + + +def patch(base_path=None, patch_file=None, patch_string=None, strip=0, output=None): + """Applies a diff from file (patch_file) or string (patch_string) + in base_path directory or current dir if None""" + + class PatchLogHandler(logging.Handler): + def __init__(self): + logging.Handler.__init__(self, logging.DEBUG) + self.output = output or ConanOutput(sys.stdout, True) + self.patchname = patch_file if patch_file else "patch" + + def emit(self, record): + logstr = self.format(record) + if record.levelno == logging.WARN: + self.output.warn("%s: %s" % (self.patchname, logstr)) + else: + self.output.info("%s: %s" % (self.patchname, logstr)) + + patchlog = logging.getLogger("patch") + if patchlog: + patchlog.handlers = [] + patchlog.addHandler(PatchLogHandler()) + + if not patch_file and not patch_string: + return + if patch_file: + patchset = fromfile(patch_file) + else: + patchset = fromstring(patch_string.encode()) + + if not patchset: + raise ConanException("Failed to parse patch: %s" % (patch_file if patch_file else "string")) + + if not patchset.apply(root=base_path, strip=strip): + raise ConanException("Failed to apply patch: %s" % patch_file) + + +def replace_in_file(file_path, search, replace, strict=True): + content = load(file_path) + if -1 == content.find(search): + message = "replace_in_file didn't find pattern '%s' in '%s' file." % (search, file_path) + if strict: + raise ConanException(message) + else: + _global_output.warn(message) + content = content.replace(search, replace) + content = content.encode("utf-8") + with open(file_path, "wb") as handle: + handle.write(content) + + +def replace_prefix_in_pc_file(pc_file, new_prefix): + content = load(pc_file) + lines = [] + for line in content.splitlines(): + if line.startswith("prefix="): + lines.append('prefix=%s' % new_prefix) + else: + lines.append(line) + save(pc_file, "\n".join(lines)) + + +def unix_path(path): + """"Used to translate windows paths to MSYS unix paths like + c/users/path/to/file""" + pattern = re.compile(r'([a-z]):\\', re.IGNORECASE) + return pattern.sub('/\\1/', path).replace('\\', '/').lower() + + +def collect_libs(conanfile, folder="lib"): + if not conanfile.package_folder: + return [] + lib_folder = os.path.join(conanfile.package_folder, folder) + if not os.path.exists(lib_folder): + conanfile.output.warn("Lib folder doesn't exist, can't collect libraries") + return [] + files = os.listdir(lib_folder) + result = [] + for f in files: + name, ext = os.path.splitext(f) + if ext in (".so", ".lib", ".a", ".dylib"): + if ext != ".lib" and name.startswith("lib"): + name = name[3:] + result.append(name) + return result \ No newline at end of file diff --git a/conans/client/tools/net.py b/conans/client/tools/net.py new file mode 100644 index 00000000000..32dd3416a33 --- /dev/null +++ b/conans/client/tools/net.py @@ -0,0 +1,48 @@ +import sys + +import os +from conans.client.output import ConanOutput +from conans.client.rest.uploader_downloader import Downloader +from conans.client.tools.files import unzip +from conans.errors import ConanException + +_global_requester = None + + +def get(url): + """ high level downloader + unziper + delete temporary zip + """ + filename = os.path.basename(url) + download(url, filename) + unzip(filename) + os.unlink(filename) + + +def ftp_download(ip, filename, login='', password=''): + import ftplib + try: + ftp = ftplib.FTP(ip, login, password) + ftp.login() + filepath, filename = os.path.split(filename) + if filepath: + ftp.cwd(filepath) + with open(filename, 'wb') as f: + ftp.retrbinary('RETR ' + filename, f.write) + except Exception as e: + raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e))) + finally: + try: + ftp.quit() + except: + pass + + +def download(url, filename, verify=True, out=None, retry=2, retry_wait=5): + out = out or ConanOutput(sys.stdout, True) + if verify: + # We check the certificate using a list of known verifiers + import conans.client.rest.cacert as cacert + verify = cacert.file_path + downloader = Downloader(_global_requester, out, verify=verify) + downloader.download(url, filename, retry=retry, retry_wait=retry_wait) + out.writeln("") diff --git a/conans/client/tools/oss.py b/conans/client/tools/oss.py new file mode 100644 index 00000000000..84a77c965d4 --- /dev/null +++ b/conans/client/tools/oss.py @@ -0,0 +1,233 @@ +import multiprocessing +import platform +import subprocess +import sys + +import os +from conans.model.version import Version +from conans.util.log import logger + +_global_output = None + + +def args_to_string(args): + if not args: + return "" + if sys.platform == 'win32': + return subprocess.list2cmdline(args) + else: + return " ".join("'" + arg.replace("'", r"'\''") + "'" for arg in args) + + +def cpu_count(): + try: + env_cpu_count = os.getenv("CONAN_CPU_COUNT", None) + return int(env_cpu_count) if env_cpu_count else multiprocessing.cpu_count() + except NotImplementedError: + _global_output.warn("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") + return 1 # Safe guess + + +def detected_architecture(): + # FIXME: Very weak check but not very common to run conan in other architectures + if "64" in platform.machine(): + return "x86_64" + elif "86" in platform.machine(): + return "x86" + return None + +# DETECT OS, VERSION AND DISTRIBUTIONS + + +class OSInfo(object): + """ Usage: + (os_info.is_linux) # True/False + (os_info.is_windows) # True/False + (os_info.is_macos) # True/False + (os_info.is_freebsd) # True/False + (os_info.is_solaris) # True/False + + (os_info.linux_distro) # debian, ubuntu, fedora, centos... + + (os_info.os_version) # 5.1 + (os_info.os_version_name) # Windows 7, El Capitan + + if os_info.os_version > "10.1": + pass + if os_info.os_version == "10.1.0": + pass + """ + + def __init__(self): + self.os_version = None + self.os_version_name = None + self.is_linux = platform.system() == "Linux" + self.linux_distro = None + self.is_windows = platform.system() == "Windows" + self.is_macos = platform.system() == "Darwin" + self.is_freebsd = platform.system() == "FreeBSD" + self.is_solaris = platform.system() == "SunOS" + + if self.is_linux: + import distro + self.linux_distro = distro.id() + self.os_version = Version(distro.version()) + version_name = distro.codename() + self.os_version_name = version_name if version_name != "n/a" else "" + if not self.os_version_name and self.linux_distro == "debian": + self.os_version_name = self.get_debian_version_name(self.os_version) + elif self.is_windows: + self.os_version = self.get_win_os_version() + self.os_version_name = self.get_win_version_name(self.os_version) + elif self.is_macos: + self.os_version = Version(platform.mac_ver()[0]) + self.os_version_name = self.get_osx_version_name(self.os_version) + elif self.is_freebsd: + self.os_version = self.get_freebsd_version() + self.os_version_name = "FreeBSD %s" % self.os_version + elif self.is_solaris: + self.os_version = Version(platform.release()) + self.os_version_name = self.get_solaris_version_name(self.os_version) + + @property + def with_apt(self): + return self.is_linux and self.linux_distro in \ + ("debian", "ubuntu", "knoppix", "linuxmint", "raspbian") + + @property + def with_yum(self): + return self.is_linux and self.linux_distro in \ + ("centos", "redhat", "fedora", "pidora", "scientific", + "xenserver", "amazon", "oracle", "rhel") + + @staticmethod + def get_win_os_version(): + """ + Get's the OS major and minor versions. Returns a tuple of + (OS_MAJOR, OS_MINOR). + """ + import ctypes + + class _OSVERSIONINFOEXW(ctypes.Structure): + _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), + ('dwMajorVersion', ctypes.c_ulong), + ('dwMinorVersion', ctypes.c_ulong), + ('dwBuildNumber', ctypes.c_ulong), + ('dwPlatformId', ctypes.c_ulong), + ('szCSDVersion', ctypes.c_wchar * 128), + ('wServicePackMajor', ctypes.c_ushort), + ('wServicePackMinor', ctypes.c_ushort), + ('wSuiteMask', ctypes.c_ushort), + ('wProductType', ctypes.c_byte), + ('wReserved', ctypes.c_byte)] + + os_version = _OSVERSIONINFOEXW() + os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) + retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) + if retcode != 0: + return None + + return Version("%d.%d" % (os_version.dwMajorVersion, os_version.dwMinorVersion)) + + @staticmethod + def get_debian_version_name(version): + if not version: + return None + elif version.major() == "8.Y.Z": + return "jessie" + elif version.major() == "7.Y.Z": + return "wheezy" + elif version.major() == "6.Y.Z": + return "squeeze" + elif version.major() == "5.Y.Z": + return "lenny" + elif version.major() == "4.Y.Z": + return "etch" + elif version.minor() == "3.1.Z": + return "sarge" + elif version.minor() == "3.0.Z": + return "woody" + + @staticmethod + def get_win_version_name(version): + if not version: + return None + elif version.major() == "5.Y.Z": + return "Windows XP" + elif version.minor() == "6.0.Z": + return "Windows Vista" + elif version.minor() == "6.1.Z": + return "Windows 7" + elif version.minor() == "6.2.Z": + return "Windows 8" + elif version.minor() == "6.3.Z": + return "Windows 8.1" + elif version.minor() == "10.0.Z": + return "Windows 10" + + @staticmethod + def get_osx_version_name(version): + if not version: + return None + elif version.minor() == "10.12.Z": + return "Sierra" + elif version.minor() == "10.11.Z": + return "El Capitan" + elif version.minor() == "10.10.Z": + return "Yosemite" + elif version.minor() == "10.9.Z": + return "Mavericks" + elif version.minor() == "10.8.Z": + return "Mountain Lion" + elif version.minor() == "10.7.Z": + return "Lion" + elif version.minor() == "10.6.Z": + return "Snow Leopard" + elif version.minor() == "10.5.Z": + return "Leopard" + elif version.minor() == "10.4.Z": + return "Tiger" + elif version.minor() == "10.3.Z": + return "Panther" + elif version.minor() == "10.2.Z": + return "Jaguar" + elif version.minor() == "10.1.Z": + return "Puma" + elif version.minor() == "10.0.Z": + return "Cheetha" + + @staticmethod + def get_freebsd_version(): + return platform.release().split("-")[0] + + @staticmethod + def get_solaris_version_name(version): + if not version: + return None + elif version.minor() == "5.10": + return "Solaris 10" + elif version.minor() == "5.11": + return "Solaris 11" + + +def cross_building(settings, self_os=None, self_arch=None): + self_os = self_os or platform.system() + self_arch = self_arch or detected_architecture() + os_setting = settings.get_safe("os") + arch_setting = settings.get_safe("arch") + platform_os = {"Darwin": "Macos"}.get(self_os, self_os) + if self_os == os_setting and self_arch == "x86_64" and arch_setting == "x86": + return False # not really considered cross + + if os_setting and platform_os != os_setting: + return True + if arch_setting and self_arch != arch_setting: + return True + + return False + +try: + os_info = OSInfo() +except Exception as exc: + logger.error(exc) + _global_output.error("Error detecting os_info") \ No newline at end of file diff --git a/conans/client/tools/system_pm.py b/conans/client/tools/system_pm.py new file mode 100644 index 00000000000..70acc43ca69 --- /dev/null +++ b/conans/client/tools/system_pm.py @@ -0,0 +1,159 @@ +import os +from conans.client.runner import ConanRunner +from conans.client.tools.oss import OSInfo +from conans.errors import ConanException + +_global_output = None + + +class SystemPackageTool(object): + + def __init__(self, runner=None, os_info=None, tool=None): + env_sudo = os.environ.get("CONAN_SYSREQUIRES_SUDO", None) + self._sudo = (env_sudo != "False" and env_sudo != "0") + os_info = os_info or OSInfo() + self._is_up_to_date = False + self._tool = tool or self._create_tool(os_info) + self._tool._sudo_str = "sudo " if self._sudo else "" + self._tool._runner = runner or ConanRunner() + + @staticmethod + def _create_tool(os_info): + if os_info.with_apt: + return AptTool() + elif os_info.with_yum: + return YumTool() + elif os_info.is_macos: + return BrewTool() + elif os_info.is_freebsd: + return PkgTool() + elif os_info.is_solaris: + return PkgUtilTool() + else: + return NullTool() + + def update(self): + """ + Get the system package tool update command + """ + self._is_up_to_date = True + self._tool.update() + + def install(self, packages, update=True, force=False): + ''' + Get the system package tool install command. + ''' + packages = [packages] if isinstance(packages, str) else list(packages) + if not force and self._installed(packages): + return + if update and not self._is_up_to_date: + self.update() + self._install_any(packages) + + def _installed(self, packages): + for pkg in packages: + if self._tool.installed(pkg): + _global_output.info("Package already installed: %s" % pkg) + return True + return False + + def _install_any(self, packages): + if len(packages) == 1: + return self._tool.install(packages[0]) + for pkg in packages: + try: + return self._tool.install(pkg) + except ConanException: + pass + raise ConanException("Could not install any of %s" % packages) + + +class NullTool(object): + def update(self): + pass + + def install(self, package_name): + _global_output.warn("Only available for linux with apt-get or yum or OSx with brew or " + "FreeBSD with pkg or Solaris with pkgutil") + + def installed(self, package_name): + return False + + +class AptTool(object): + def update(self): + _run(self._runner, "%sapt-get update" % self._sudo_str) + + def install(self, package_name): + _run(self._runner, "%sapt-get install -y %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner("dpkg -s %s" % package_name, None) + return exit_code == 0 + + +class YumTool(object): + def update(self): + _run(self._runner, "%syum check-update" % self._sudo_str, accepted_returns=[0, 100]) + + def install(self, package_name): + _run(self._runner, "%syum install -y %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner("rpm -q %s" % package_name, None) + return exit_code == 0 + + +class BrewTool(object): + def update(self): + _run(self._runner, "brew update") + + def install(self, package_name): + _run(self._runner, "brew install %s" % package_name) + + def installed(self, package_name): + exit_code = self._runner('test -n "$(brew ls --versions %s)"' % package_name, None) + return exit_code == 0 + + +class PkgTool(object): + def update(self): + _run(self._runner, "%spkg update" % self._sudo_str) + + def install(self, package_name): + _run(self._runner, "%spkg install -y %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner("pkg info %s" % package_name, None) + return exit_code == 0 + + +class PkgUtilTool(object): + def update(self): + _run(self._runner, "%spkgutil --catalog" % self._sudo_str) + + def install(self, package_name): + _run(self._runner, "%spkgutil --install --yes %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner('test -n "`pkgutil --list %s`"' % package_name, None) + return exit_code == 0 + + +class ChocolateyTool(object): + def update(self): + _run(self._runner, "choco outdated") + + def install(self, package_name): + _run(self._runner, "choco install --yes %s" % package_name) + + def installed(self, package_name): + exit_code = self._runner('choco search --local-only --exact %s | findstr /c:"1 packages installed."' % package_name, None) + return exit_code == 0 + + +def _run(runner, command, accepted_returns=None): + accepted_returns = accepted_returns or [0, ] + _global_output.info("Running: %s" % command) + if runner(command, True) not in accepted_returns: + raise ConanException("Command '%s' failed" % command) diff --git a/conans/client/tools/win.py b/conans/client/tools/win.py new file mode 100644 index 00000000000..f15c5182435 --- /dev/null +++ b/conans/client/tools/win.py @@ -0,0 +1,150 @@ +import os +import platform + +import subprocess + +from conans.client.tools.env import environment_append +from conans.client.tools.files import unix_path +from conans.errors import ConanException + +_global_output = None + + +def msvc_build_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, + arch=None): + """ Do both: set the environment variables and call the .sln build + """ + vcvars = vcvars_command(settings) + build = build_sln_command(settings, sln_path, targets, upgrade_project, build_type, arch) + command = "%s && %s" % (vcvars, build) + return command + + +def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, + arch=None): + """ + Use example: + build_command = build_sln_command(self.settings, "myfile.sln", targets=["SDL2_image"]) + command = "%s && %s" % (tools.vcvars_command(self.settings), build_command) + self.run(command) + """ + targets = targets or [] + command = "devenv %s /upgrade && " % sln_path if upgrade_project else "" + build_type = build_type or settings.build_type + arch = arch or settings.arch + if not build_type: + raise ConanException("Cannot build_sln_command, build_type not defined") + if not arch: + raise ConanException("Cannot build_sln_command, arch not defined") + command += "msbuild %s /p:Configuration=%s" % (sln_path, build_type) + arch = str(arch) + if arch in ["x86_64", "x86"]: + command += ' /p:Platform=' + command += '"x64"' if arch == "x86_64" else '"x86"' + elif "ARM" in arch.upper(): + command += ' /p:Platform="ARM"' + + if targets: + command += " /target:%s" % ";".join(targets) + return command + + +def vs_installation_path(version): + if not hasattr(vs_installation_path, "_cached"): + vs_installation_path._cached = dict() + + if version not in vs_installation_path._cached: + vs_path = None + program_files = os.environ.get("ProgramFiles(x86)", os.environ.get("ProgramFiles")) + if program_files: + vswhere_path = os.path.join(program_files, "Microsoft Visual Studio", "Installer", + "vswhere.exe") + if os.path.isfile(vswhere_path): + version_range = "[%d.0, %d.0)" % (int(version), int(version) + 1) + try: + output = subprocess.check_output([vswhere_path, "-version", version_range, + "-legacy", "-property", "installationPath"]) + vs_path = output.decode().strip() + _global_output.info("vswhere detected VS %s in %s" % (version, vs_path)) + except (ValueError, subprocess.CalledProcessError, UnicodeDecodeError) as e: + _global_output.error("vswhere error: %s" % str(e)) + + # Remember to cache result + vs_installation_path._cached[version] = vs_path + + return vs_installation_path._cached[version] + + +def vcvars_command(settings): + arch_setting = settings.get_safe("arch") + compiler_version = settings.get_safe("compiler.version") + if not compiler_version: + raise ConanException("compiler.version setting required for vcvars not defined") + + param = "x86" if arch_setting == "x86" else "amd64" + existing_version = os.environ.get("VisualStudioVersion") + if existing_version: + command = "echo Conan:vcvars already set" + existing_version = existing_version.split(".")[0] + if existing_version != compiler_version: + raise ConanException("Error, Visual environment already set to %s\n" + "Current settings visual version: %s" + % (existing_version, compiler_version)) + else: + env_var = "vs%s0comntools" % compiler_version + + if env_var == 'vs150comntools': + vs_path = os.getenv(env_var) + if not vs_path: # Try to locate with vswhere + vs_root = vs_installation_path("15") + if vs_root: + vs_path = os.path.join(vs_root, "Common7", "Tools") + else: + raise ConanException("VS2017 '%s' variable not defined, " + "and vswhere didn't find it" % env_var) + vcvars_path = os.path.join(vs_path, "../../VC/Auxiliary/Build/vcvarsall.bat") + command = ('set "VSCMD_START_DIR=%%CD%%" && ' + 'call "%s" %s' % (vcvars_path, param)) + else: + try: + vs_path = os.environ[env_var] + except KeyError: + raise ConanException("VS '%s' variable not defined. Please install VS" % env_var) + vcvars_path = os.path.join(vs_path, "../../VC/vcvarsall.bat") + command = ('call "%s" %s' % (vcvars_path, param)) + + return command + + +def escape_windows_cmd(command): + """ To use in a regular windows cmd.exe + 1. Adds escapes so the argument can be unpacked by CommandLineToArgvW() + 2. Adds escapes for cmd.exe so the argument survives cmd.exe's substitutions. + + Useful to escape commands to be executed in a windows bash (msys2, cygwin etc) + """ + quoted_arg = subprocess.list2cmdline([command]) + return "".join(["^%s" % arg if arg in r'()%!^"<>&|' else arg for arg in quoted_arg]) + + +def run_in_windows_bash(conanfile, bashcmd, cwd=None): + """ Will run a unix command inside the msys2 environment + It requires to have MSYS2 in the path and MinGW + """ + if platform.system() != "Windows": + raise ConanException("Command only for Windows operating system") + # This needs to be set so that msys2 bash profile will set up the environment correctly. + try: + arch = conanfile.settings.arch # Maybe arch doesn't exist + except: + arch = None + env_vars = {"MSYSTEM": "MINGW32" if arch == "x86" else "MINGW64", + "MSYS2_PATH_TYPE": "inherit"} + with environment_append(env_vars): + curdir = unix_path(cwd or os.path.abspath(os.path.curdir)) + # Needed to change to that dir inside the bash shell + to_run = 'cd "%s" && %s ' % (curdir, bashcmd) + custom_bash_path = os.getenv("CONAN_BASH_PATH", "bash") + wincmd = '%s --login -c %s' % (custom_bash_path, escape_windows_cmd(to_run)) + conanfile.output.info('run_in_windows_bash: %s' % wincmd) + conanfile.run(wincmd) diff --git a/conans/errors.py b/conans/errors.py index 66d717dc436..0fcbadff53e 100644 --- a/conans/errors.py +++ b/conans/errors.py @@ -8,9 +8,10 @@ see return_plugin.py """ - from contextlib import contextmanager +from conans.util.env_reader import get_env + @contextmanager def conanfile_exception_formatter(conanfile_name, func_name): @@ -27,25 +28,40 @@ def conanfile_exception_formatter(conanfile_name, func_name): def _format_conanfile_exception(scope, method, exception): + """ + It will iterate the traceback lines, when it finds that the source code is inside the users + conanfile it "start recording" the messages, when the trace exits the conanfile we return + the traces. + """ import sys import traceback - msg = "%s: Error in %s() method" % (scope, method) + if get_env("CONAN_VERBOSE_TRACEBACK", False): + return traceback.format_exc() try: + conanfile_reached = False tb = sys.exc_info()[2] - index = -1 + index = 0 + content_lines = [] + while True: # If out of index will raise and will be captured later filepath, line, name, contents = traceback.extract_tb(tb, 40)[index] # 40 levels of nested functions max, get the latest - if not "conanfile.py" in filepath: # Avoid show trace from internal conan source code - index -= 1 + if "conanfile.py" not in filepath: # Avoid show trace from internal conan source code + if conanfile_reached: # The error goes to internal code, exit print + break else: - break - if name != method: - msg += ", while calling '%s'" % name - msg += ", line %d\n\t%s" % (line, contents) if line else "\n\t%s" % contents + if not conanfile_reached: # First line + msg = "%s: Error in %s() method" % (scope, method) + msg += ", line %d\n\t%s" % (line, contents) + else: + msg = "while calling '%s', line %d\n\t%s" % (name, line, contents) if line else "\n\t%s" % contents + content_lines.append(msg) + conanfile_reached = True + index += 1 except: pass - msg += "\n\t%s: %s" % (exception.__class__.__name__, str(exception)) - return msg + ret = "\n".join(content_lines) + ret += "\n\t%s: %s" % (exception.__class__.__name__, str(exception)) + return ret class ConanException(Exception): @@ -70,8 +86,8 @@ class ConanOutdatedClient(ConanException): class ConanExceptionInUserConanfileMethod(ConanException): pass -# Remote exceptions # +# Remote exceptions # class InternalErrorException(ConanException): """ Generic 500 error diff --git a/conans/model/__init__.py b/conans/model/__init__.py index 0e6f2ac3e9a..b7a948e27df 100644 --- a/conans/model/__init__.py +++ b/conans/model/__init__.py @@ -1,3 +1 @@ -from .conan_generator import GeneratorManager, Generator - -registered_generators = GeneratorManager() +from .conan_generator import Generator diff --git a/conans/model/build_info.py b/conans/model/build_info.py index 92133bb5995..fed762aa936 100644 --- a/conans/model/build_info.py +++ b/conans/model/build_info.py @@ -27,31 +27,46 @@ def __init__(self): self.exelinkflags = [] # linker flags self.rootpath = "" self.sysroot = None + self._include_paths = None + self._lib_paths = None + self._bin_paths = None + self._build_paths = None + self._res_paths = None + + def _filter_paths(self, paths): + abs_paths = [os.path.join(self.rootpath, p) + if not os.path.isabs(p) else p for p in paths] + return [p for p in abs_paths if os.path.isdir(p)] @property def include_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.includedirs] + if self._include_paths is None: + self._include_paths = self._filter_paths(self.includedirs) + return self._include_paths @property def lib_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.libdirs] + if self._lib_paths is None: + self._lib_paths = self._filter_paths(self.libdirs) + return self._lib_paths @property def bin_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.bindirs] + if self._bin_paths is None: + self._bin_paths = self._filter_paths(self.bindirs) + return self._bin_paths @property def build_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.builddirs] + if self._build_paths is None: + self._build_paths = self._filter_paths(self.builddirs) + return self._build_paths @property def res_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.resdirs] + if self._res_paths is None: + self._res_paths = self._filter_paths(self.resdirs) + return self._res_paths class CppInfo(_CppInfo): diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index a24ce5e93f2..2a4db5fcdd7 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -130,6 +130,9 @@ def __init__(self, output, runner, settings, conanfile_directory, user=None, cha self._user = user self._channel = channel + # Are we in local cache? Suggest a better name + self.in_local_cache = False + @property def env(self): simple, multiple = self._env_values.env_dicts(self.name) @@ -155,21 +158,9 @@ def user(self): return self._user def collect_libs(self, folder="lib"): - if not self.package_folder: - return [] - lib_folder = os.path.join(self.package_folder, folder) - if not os.path.exists(lib_folder): - self.output.warn("Lib folder doesn't exist, can't collect libraries") - return [] - files = os.listdir(lib_folder) - result = [] - for f in files: - name, ext = os.path.splitext(f) - if ext in (".so", ".lib", ".a", ".dylib"): - if ext != ".lib" and name.startswith("lib"): - name = name[3:] - result.append(name) - return result + self.output.warn("Use 'self.collect_libs' is deprecated, " + "use tools.collect_libs(self) instead") + return tools.collect_libs(self, folder=folder) @property def scope(self): @@ -178,16 +169,6 @@ def scope(self): @scope.setter def scope(self, value): self._scope = value - if value.dev: - self.requires.allow_dev = True - try: - if hasattr(self, "dev_requires"): - if isinstance(self.dev_requires, tuple): - self.requires.add_dev(*self.dev_requires) - else: - self.requires.add_dev(self.dev_requires, ) - except Exception as e: - raise ConanException("Error while initializing dev_requirements. %s" % str(e)) @property def conanfile_directory(self): diff --git a/conans/model/conan_generator.py b/conans/model/conan_generator.py index f81a236d5c3..1a5c8195134 100644 --- a/conans/model/conan_generator.py +++ b/conans/model/conan_generator.py @@ -1,4 +1,3 @@ -from conans.errors import ConanException from abc import ABCMeta, abstractproperty @@ -24,11 +23,11 @@ def build_info(self): @property def deps_env_info(self): return self._deps_env_info - + @property def deps_user_info(self): return self._deps_user_info - + @property def env_info(self): return self._env_info @@ -44,30 +43,3 @@ def content(self): @abstractproperty def filename(self): raise NotImplementedError() - - -class GeneratorManager(object): - def __init__(self): - self._known_generators = {} - - def add(self, name, generator_class): - if name in self._known_generators: - raise ConanException("") - elif not issubclass(generator_class, Generator): - raise ConanException("") - else: - self._known_generators[name] = generator_class - - def remove(self, name): - if name in self._known_generators: - del self._known_generators[name] - - @property - def available(self): - return list(self._known_generators.keys()) - - def __contains__(self, key): - return key in self._known_generators - - def __getitem__(self, key): - return self._known_generators[key] diff --git a/conans/model/info.py b/conans/model/info.py index 7cfca9e29d0..fecae68326d 100644 --- a/conans/model/info.py +++ b/conans/model/info.py @@ -101,14 +101,12 @@ def full_package_mode(self): class RequirementsInfo(object): - def __init__(self, requires, non_devs_requirements): + def __init__(self, requires): # {PackageReference: RequirementInfo} - self._non_devs_requirements = non_devs_requirements self._data = {r: RequirementInfo(str(r)) for r in requires} def copy(self): - return RequirementsInfo(self._data.keys(), self._non_devs_requirements.copy() - if self._non_devs_requirements else None) + return RequirementsInfo(self._data.keys()) def clear(self): self._data = {} @@ -151,14 +149,8 @@ def sha(self): result = [] # Remove requirements without a name, i.e. indirect transitive requirements data = {k: v for k, v in self._data.items() if v.name} - if self._non_devs_requirements is None: - for key in sorted(data): - result.append(data[key].sha) - else: - for key in sorted(data): - non_dev = key.conan.name in self._non_devs_requirements - if non_dev: - result.append(data[key].sha) + for key in sorted(data): + result.append(data[key].sha) return sha1('\n'.join(result).encode()) def dumps(self): @@ -166,10 +158,6 @@ def dumps(self): for ref in sorted(self._data): dumped = self._data[ref].dumps() if dumped: - dev = (self._non_devs_requirements is not None and - ref.conan.name not in self._non_devs_requirements) - if dev: - dumped += " DEV" result.append(dumped) return "\n".join(result) @@ -245,11 +233,10 @@ def copy(self): result.settings = self.settings.copy() result.options = self.options.copy() result.requires = self.requires.copy() - result._non_devs_requirements = self._non_devs_requirements return result @staticmethod - def create(settings, options, requires, indirect_requires, non_devs_requirements): + def create(settings, options, requires, indirect_requires): result = ConanInfo() result.full_settings = settings result.settings = settings.copy() @@ -257,12 +244,11 @@ def create(settings, options, requires, indirect_requires, non_devs_requirements result.options = options.copy() result.options.clear_indirect() result.full_requires = RequirementsList(requires) - result.requires = RequirementsInfo(requires, non_devs_requirements) + result.requires = RequirementsInfo(requires) result.scope = None result.requires.add(indirect_requires) result.full_requires.extend(indirect_requires) result.recipe_hash = None - result._non_devs_requirements = non_devs_requirements # Can be None result.env_values = EnvValues() return result @@ -277,7 +263,7 @@ def loads(text): result.options = OptionsValues.loads(parser.options) result.full_options = OptionsValues.loads(parser.full_options) result.full_requires = RequirementsList.loads(parser.full_requires) - result.requires = RequirementsInfo(result.full_requires, None) + result.requires = RequirementsInfo(result.full_requires) result.recipe_hash = parser.recipe_hash or None # TODO: Missing handling paring of requires, but not necessary now @@ -344,7 +330,7 @@ def package_id(self): # Only are valid requires for OPtions those Non-Dev who are still in requires self.options.filter_used(self.requires.pkg_names) - result.append(self.options.sha(self._non_devs_requirements)) + result.append(self.options.sha) result.append(self.requires.sha) self._package_id = sha1('\n'.join(result).encode()) return self._package_id diff --git a/conans/model/options.py b/conans/model/options.py index 54d43672fe0..b3b7843f20e 100644 --- a/conans/model/options.py +++ b/conans/model/options.py @@ -254,17 +254,12 @@ def loads(text): result.append((name.strip(), value.strip())) return OptionsValues(result) - def sha(self, non_dev_requirements): + @property + def sha(self): result = [] result.append(self._package_values.sha) - if non_dev_requirements is None: # Not filtering - for key in sorted(list(self._reqs_options.keys())): - result.append(self._reqs_options[key].sha) - else: - for key in sorted(list(self._reqs_options.keys())): - non_dev = key in non_dev_requirements - if non_dev: - result.append(self._reqs_options[key].sha) + for key in sorted(list(self._reqs_options.keys())): + result.append(self._reqs_options[key].sha) return sha1('\n'.join(result).encode()) def serialize(self): diff --git a/conans/model/profile.py b/conans/model/profile.py index c702a861bcc..b1825947193 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -82,7 +82,7 @@ def update_settings(self, new_settings): # Example: new_settings declare a different "compiler", so invalidate the current "compiler.XXX" for name, value in new_settings.items(): if "." not in name: - if name in self.settings and self.settings[name] != new_settings[name]: + if name in self.settings and self.settings[name] != value: for cur_name, _ in self.settings.items(): if cur_name.startswith(name): del res[cur_name] diff --git a/conans/model/requires.py b/conans/model/requires.py index 5c7a221262e..ad7e1ae6ba5 100644 --- a/conans/model/requires.py +++ b/conans/model/requires.py @@ -8,20 +8,15 @@ class Requirement(object): """ A reference to a package plus some attributes of how to depend on that package """ - def __init__(self, conan_reference, private=False, override=False, dev=False): + def __init__(self, conan_reference, private=False, override=False): """ param override: True means that this is not an actual requirement, but something to be passed upstream and override possible existing values - param private: True means that this requirement will be somewhat embedded (like - a static lib linked into a shared lib), so it is not required to link - param dev: True means that this requirement is only needed at dev time, e.g. only - needed for building or testing, but not affects the package hash at all """ self.conan_reference = conan_reference self.range_reference = conan_reference - self.private = private self.override = override - self.dev = dev + self.private = private @property def version_range(self): @@ -45,8 +40,7 @@ def __repr__(self): def __eq__(self, other): return (self.override == other.override and self.conan_reference == other.conan_reference and - self.private == other.private and - self.dev == other.dev) + self.private == other.private) def __ne__(self, other): return not self.__eq__(other) @@ -58,23 +52,6 @@ class Requirements(OrderedDict): def __init__(self, *args): super(Requirements, self).__init__() - self.allow_dev = False - for v in args: - if isinstance(v, tuple): - override = private = dev = False - ref = v[0] - for elem in v[1:]: - if elem == "override": - override = True - elif elem == "private": - private = True - else: - raise ConanException("Unknown requirement config %s" % elem) - self.add(ref, private=private, override=override, dev=dev) - else: - self.add(v) - - def add_dev(self, *args): for v in args: if isinstance(v, tuple): override = private = False @@ -86,9 +63,9 @@ def add_dev(self, *args): private = True else: raise ConanException("Unknown requirement config %s" % elem) - self.add(ref, private=private, override=override, dev=True) + self.add(ref, private=private, override=override) else: - self.add(v, dev=True) + self.add(v) def copy(self): """ We need a custom copy as the normal one requires __init__ to be @@ -103,17 +80,15 @@ def copy(self): def iteritems(self): # FIXME: Just a trick to not change default testing conanfile for py3 return self.items() - def add(self, reference, private=False, override=False, dev=False): + def add(self, reference, private=False, override=False): """ to define requirements by the user in text, prior to any propagation """ assert isinstance(reference, six.string_types) - if dev and not self.allow_dev: - return conan_reference = ConanFileReference.loads(reference) name = conan_reference.name - new_requirement = Requirement(conan_reference, private, override, dev) + new_requirement = Requirement(conan_reference, private, override) old_requirement = self.get(name) if old_requirement and old_requirement != new_requirement: raise ConanException("Duplicated requirement %s != %s" @@ -138,7 +113,7 @@ def update(self, down_reqs, output, own_ref, down_ref): if own_ref: new_reqs.pop(own_ref.name, None) for name, req in self.items(): - if req.private or req.dev: + if req.private: continue if name in down_reqs: other_req = down_reqs[name] @@ -153,8 +128,8 @@ def update(self, down_reqs, output, own_ref, down_ref): new_reqs[name] = req return new_reqs - def __call__(self, conan_reference, private=False, override=False, dev=False): - self.add(conan_reference, private, override, dev) + def __call__(self, conan_reference, private=False, override=False): + self.add(conan_reference, private, override) def __repr__(self): result = [] diff --git a/conans/model/settings.py b/conans/model/settings.py index ffc3ab707da..38f83fe7b82 100644 --- a/conans/model/settings.py +++ b/conans/model/settings.py @@ -40,6 +40,9 @@ def __init__(self, definition, name): # list or tuple of possible values self._definition = sorted(str(v) for v in definition) + def __contains__(self, value): + return value in (self._value or "") + def copy(self): """ deepcopy, recursive """ diff --git a/conans/paths.py b/conans/paths.py index 2f6e1324dd5..eb028b3c8fc 100644 --- a/conans/paths.py +++ b/conans/paths.py @@ -1,10 +1,17 @@ import os from conans.model.ref import ConanFileReference, PackageReference -from conans.util.files import load, save, rmdir from os.path import join, normpath import platform -import tempfile from conans.errors import ConanException +from conans.util.files import rmdir + +if platform.system() == "Windows": + from conans.util.windows import path_shortener, rm_conandir, conan_expand_user +else: + def path_shortener(x, _): + return x + conan_expand_user = os.path.expanduser + rm_conandir = rmdir EXPORT_FOLDER = "export" @@ -29,46 +36,18 @@ CONANINFO = "conaninfo.txt" CONANENV = "conanenv.txt" SYSTEM_REQS = "system_reqs.txt" -DIRTY_FILE = ".conan_dirty" PUT_HEADERS = "artifacts.properties" PACKAGE_TGZ_NAME = "conan_package.tgz" EXPORT_TGZ_NAME = "conan_export.tgz" EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz" -EXPORT_SOURCES_DIR = ".c_src" -CONAN_LINK = ".conan_link" +EXPORT_SOURCES_DIR_OLD = ".c_src" RUN_LOG_NAME = "conan_run.log" DEFAULT_PROFILE_NAME = "default" -def conan_expand_user(path): - """ wrapper to the original expanduser function, to workaround python returning - verbatim %USERPROFILE% when some other app (git for windows) sets HOME envvar - """ - if platform.system() == "Windows": - # In win these variables should exist and point to user directory, which - # must exist. Using context to avoid permanent modification of os.environ - old_env = dict(os.environ) - try: - home = os.environ.get("HOME") - # Problematic cases of wrong HOME variable - # - HOME = %USERPROFILE% verbatim, as messed by some other tools - # - MSYS console, that defines a different user home in /c/mingw/msys/users/xxx - # In these cases, it is safe to remove it and rely on USERPROFILE directly - if home and (not os.path.exists(home) or - (os.getenv("MSYSTEM") and os.getenv("USERPROFILE"))): - del os.environ["HOME"] - result = os.path.expanduser(path) - finally: - os.environ.clear() - os.environ.update(old_env) - return result - - return os.path.expanduser(path) - - def get_conan_user_home(): tmp = conan_expand_user(os.getenv("CONAN_USER_HOME", "~")) if not os.path.isabs(tmp): @@ -78,19 +57,6 @@ def get_conan_user_home(): return os.path.abspath(tmp) -if platform.system() == "Windows": - def _rm_conandir(path): - """removal of a directory that might contain a link to a short path""" - link = os.path.join(path, CONAN_LINK) - if os.path.exists(link): - short_path = load(link) - rmdir(os.path.dirname(short_path)) - rmdir(path) - rm_conandir = _rm_conandir -else: - rm_conandir = rmdir - - def is_case_insensitive_os(): system = platform.system() return system != "Linux" and system != "FreeBSD" and system != "SunOS" @@ -119,41 +85,6 @@ def _check_ref_case(conan_reference, conan_folder, store_folder): # @UnusedVari pass -def _shortener(path, short_paths): - """ short_paths is 4-state: - False: Never shorten the path - True: Always shorten the path, create link if not existing - None: Use shorten path only if already exists, not create - Other: Integrity check. Consumer knows it should be short, but it isn't - """ - if short_paths is False: - return path - link = os.path.join(path, CONAN_LINK) - if os.path.exists(link): - return load(link) - elif short_paths is None: - return path - elif short_paths is not True: - raise ConanException("This path should be short, but it isn't: %s\n" - "Try to remove these packages and re-build them" % path) - - short_home = os.getenv("CONAN_USER_HOME_SHORT") - if not short_home: - drive = os.path.splitdrive(path)[0] - short_home = drive + "/.conan" - try: - os.makedirs(short_home) - except: - pass - redirect = tempfile.mkdtemp(dir=short_home, prefix="") - # This "1" is the way to have a non-existing directory, so commands like - # shutil.copytree() to it, works. It can be removed without compromising the - # temp folder generator and conan-links consistency - redirect = os.path.join(redirect, "1") - save(link, redirect) - return redirect - - class SimplePaths(object): """ Generate Conan paths. Handles the conan domain path logic. NO DISK ACCESS, just @@ -161,10 +92,6 @@ class SimplePaths(object): """ def __init__(self, store_folder): self._store_folder = store_folder - if platform.system() == "Windows": - self._shortener = _shortener - else: - self._shortener = lambda x, _: x @property def store(self): @@ -183,12 +110,12 @@ def export(self, conan_reference): def export_sources(self, conan_reference, short_paths=False): assert isinstance(conan_reference, ConanFileReference) p = normpath(join(self.conan(conan_reference), EXPORT_SRC_FOLDER)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) def source(self, conan_reference, short_paths=False): assert isinstance(conan_reference, ConanFileReference) p = normpath(join(self.conan(conan_reference), SRC_FOLDER)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) def conanfile(self, conan_reference): export = self.export(conan_reference) @@ -212,7 +139,7 @@ def build(self, package_reference, short_paths=False): assert isinstance(package_reference, PackageReference) p = normpath(join(self.conan(package_reference.conan), BUILD_FOLDER, package_reference.package_id)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) def system_reqs(self, conan_reference): assert isinstance(conan_reference, ConanFileReference) @@ -231,4 +158,4 @@ def package(self, package_reference, short_paths=False): assert isinstance(package_reference, PackageReference) p = normpath(join(self.conan(package_reference.conan), PACKAGES_FOLDER, package_reference.package_id)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) diff --git a/conans/requirements.txt b/conans/requirements.txt index 9f6676a975c..7a3148eb7ca 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -5,9 +5,9 @@ PyYAML>=3.11, <3.13.0 patch==1.16 fasteners>=0.14.1 six>=1.10.0 -node-semver==0.1.1 +node-semver==0.2.0 distro>=1.0.2, <1.1.0 -pylint==1.6.5 +pylint>=1.6.5, <=1.8.0 future==0.16.0 pygments>=2.0, <3.0 diff --git a/conans/server/conf/__init__.py b/conans/server/conf/__init__.py index 48e06b3f5de..5d6fff511a6 100644 --- a/conans/server/conf/__init__.py +++ b/conans/server/conf/__init__.py @@ -16,7 +16,7 @@ from conans.util.log import logger from conans.server.conf.default_server_conf import default_server_conf -MIN_CLIENT_COMPATIBLE_VERSION = '0.19.3' +MIN_CLIENT_COMPATIBLE_VERSION = '0.25.0' class ConanServerConfigParser(ConfigParser): diff --git a/conans/server/conf/default_server_conf.py b/conans/server/conf/default_server_conf.py index b846e597e9d..6b3f232d51e 100644 --- a/conans/server/conf/default_server_conf.py +++ b/conans/server/conf/default_server_conf.py @@ -45,10 +45,15 @@ # name/version@user/channel: user1, user2, user3 # The rules are applied in order. If a rule applies to a conan, system wont look further. # -# Example: All versions of opencv package from lasote user in testing channel is only -# readable by default_user and default_user2. Rest of packages are world readable +# Example: +# All versions of opencv package from lasote user in testing channel are only +# readable by default_user and default_user2. +# All versions of internal package from any user/channel are only readable by +# authenticated users. +# Rest of packages are world readable. # -# opencv/1.2.3@lasote/testing: default_user default_user2 +# opencv/*@lasote/testing: default_user default_user2 +# internal/*@*/*: ? # *:*@*/*: * # # By default all users can read all blocks diff --git a/conans/server/rest/controllers/file_upload_download_controller.py b/conans/server/rest/controllers/file_upload_download_controller.py index 7552f41a994..b49091a0798 100644 --- a/conans/server/rest/controllers/file_upload_download_controller.py +++ b/conans/server/rest/controllers/file_upload_download_controller.py @@ -4,7 +4,6 @@ import os from unicodedata import normalize import six -from conans.errors import NotFoundException class FileUploadDownloadController(Controller): @@ -35,7 +34,6 @@ def put(filepath): abs_path = os.path.abspath(os.path.join(storage_path, os.path.normpath(filepath))) # Body is a stringIO (generator) service.put_file(file_saver, abs_path, token, request.content_length) - return class ConanFileUpload(FileUpload): diff --git a/conans/server/service/authorize.py b/conans/server/service/authorize.py index 7223729277e..fb531661f4c 100644 --- a/conans/server/service/authorize.py +++ b/conans/server/service/authorize.py @@ -189,7 +189,10 @@ def _check_rule_ok(self, username, rule, conan_reference): return True # Ok, applies and match username else: if username: - raise ForbiddenException("Permission denied") + if authorized_users[0] == "?": + return True #Ok, applies and match any authenticated username + else: + raise ForbiddenException("Permission denied") else: raise AuthenticationException() diff --git a/conans/server/service/service.py b/conans/server/service/service.py index 238df10bf09..592a85430f8 100644 --- a/conans/server/service/service.py +++ b/conans/server/service/service.py @@ -47,7 +47,7 @@ def put_file(self, file_saver, abs_filepath, token, upload_size): file_saver.save(os.path.dirname(abs_filepath)) except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError): - return NotFoundException("File not found") + raise NotFoundException("File not found") def _valid_path(self, filepath, encoded_path): if encoded_path == filepath: diff --git a/conans/test/command/build_test.py b/conans/test/command/build_test.py index 18eb7fbbfbe..a7f90a37b10 100644 --- a/conans/test/command/build_test.py +++ b/conans/test/command/build_test.py @@ -21,10 +21,15 @@ def build(self): conanfile_dep = """ from conans import ConanFile +from conans.tools import mkdir +import os class AConan(ConanFile): name = "Hello" version = "0.1" + + def package(self): + mkdir(os.path.join(self.package_folder, "include")) """ @@ -80,6 +85,35 @@ def build_test(self): self.assertIn("Project: HELLO INCLUDE PATHS: %s/include" % package_folder, client.user_io.out) + def build_dots_names_test(self): + """ Try to reuse variables loaded from txt generator => deps_cpp_info + """ + client = TestClient() + conanfile_dep = """ +from conans import ConanFile + +class AConan(ConanFile): + pass +""" + client.save({CONANFILE: conanfile_dep}) + client.run("create Hello.Pkg/0.1@lasote/testing") + client.run("create Hello-Tools/0.1@lasote/testing") + conanfile_scope_env = """ +from conans import ConanFile + +class AConan(ConanFile): + requires = "Hello.Pkg/0.1@lasote/testing", "Hello-Tools/0.1@lasote/testing" + + def build(self): + self.output.info("HELLO ROOT PATH: %s" % self.deps_cpp_info["Hello.Pkg"].rootpath) + self.output.info("HELLO ROOT PATH: %s" % self.deps_cpp_info["Hello-Tools"].rootpath) +""" + client.save({CONANFILE: conanfile_scope_env}, clean_first=True) + client.run("install --build=missing -g txt") + client.run("build") + self.assertIn("Hello.Pkg/0.1/lasote/testing", client.out) + self.assertIn("Hello-Tools/0.1/lasote/testing", client.out) + def build_cmake_install_test(self): client = TestClient() conanfile = """ diff --git a/conans/test/functional/conan_get_test.py b/conans/test/command/conan_get_test.py similarity index 89% rename from conans/test/functional/conan_get_test.py rename to conans/test/command/conan_get_test.py index edc5ffd41fd..a5c74c5d6ba 100644 --- a/conans/test/functional/conan_get_test.py +++ b/conans/test/command/conan_get_test.py @@ -87,7 +87,8 @@ def test_get_remote(self): # Remote search, dir list self.client.run('get Hello0/0.1@lasote/channel . -r default --raw') - self.assertIn("conan_export.tgz\nconan_sources.tgz\nconanfile.py\nconanmanifest.txt", self.client.user_io.out) + self.assertIn("conan_export.tgz\nconan_sources.tgz\nconanfile.py\nconanmanifest.txt", + self.client.user_io.out) # Remote search, conanfile print self.client.run('get Hello0/0.1@lasote/channel -r default --raw') @@ -95,12 +96,15 @@ def test_get_remote(self): # List package dir self.client.run('get Hello0/0.1@lasote/channel "." -p 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 --raw -r default') - self.assertEquals("conan_package.tgz\nconaninfo.txt\nconanmanifest.txt\n", self.client.user_io.out) + self.assertEquals("conan_package.tgz\nconaninfo.txt\nconanmanifest.txt\n", + self.client.user_io.out) def test_not_found(self): self.client.run('get Hello0/0.1@lasote/channel "." -r default', ignore_error=True) self.assertIn("Recipe Hello0/0.1@lasote/channel not found", self.client.user_io.out) - self.client.run('get Hello0/0.1@lasote/channel "." -r default -p 123123123123123', ignore_error=True) - self.assertIn("Package Hello0/0.1@lasote/channel:123123123123123 not found", self.client.user_io.out) - + error = self.client.run('get Hello0/0.1@lasote/channel "." -r default -p 123123123123123', + ignore_error=True) + self.assertTrue(error) + self.assertIn("Package Hello0/0.1@lasote/channel:123123123123123 not found", + self.client.user_io.out) diff --git a/conans/test/command/config_install_test.py b/conans/test/command/config_install_test.py new file mode 100644 index 00000000000..983e90999c0 --- /dev/null +++ b/conans/test/command/config_install_test.py @@ -0,0 +1,187 @@ +import unittest + +from conans.test.utils.tools import TestClient, TestBufferConanOutput +import os +import zipfile +from conans.test.utils.test_files import temp_folder +from conans.util.files import load, save_files, save +from conans.client.remote_registry import RemoteRegistry, Remote +from mock import patch +from conans.client.rest.uploader_downloader import Downloader +from conans import tools +from conans.client.conf import ConanClientConfigParser +import shutil + + +win_profile = """[settings] + os: Windows +""" + +linux_profile = """[settings] + os: Linux +""" + +remotes = """myrepo1 https://myrepourl.net False +my-repo-2 https://myrepo2.com True +""" + +registry = """myrepo1 https://myrepourl.net False + +Pkg/1.0@user/channel myrepo1 +""" + +settings_yml = """os: + Windows: + Linux: +arch: [x86, x86_64] +""" + +conan_conf = """ +[log] +run_to_output = False # environment CONAN_LOG_RUN_TO_OUTPUT +level = 10 # environment CONAN_LOGGING_LEVEL + +[general] +compression_level = 6 # environment CONAN_COMPRESSION_LEVEL +cpu_count = 1 # environment CONAN_CPU_COUNT + +[proxies] +# Empty section will try to use system proxies. +# If don't want proxy at all, remove section [proxies] +# As documented in http://docs.python-requests.org/en/latest/user/advanced/#proxies +http = http://user:pass@10.10.1.10:3128/ +no_proxy = mylocalhost +https = None +# http = http://10.10.1.10:3128 +# https = http://10.10.1.10:1080 +""" + + +def zipdir(path, zipfilename): + with zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_DEFLATED) as z: + for root, _, files in os.walk(path): + for f in files: + z.write(os.path.join(root, f)) + + +class ConfigInstallTest(unittest.TestCase): + + def setUp(self): + self.client = TestClient() + registry_path = self.client.client_cache.registry + + save(registry_path, """my-repo-2 https://myrepo2.com True +conan-center https://conan-center.com + +MyPkg/0.1@user/channel my-repo-2 +Other/1.2@user/channel conan-center +""") + save(os.path.join(self.client.client_cache.profiles_path, "default"), "#default profile empty") + save(os.path.join(self.client.client_cache.profiles_path, "linux"), "#empty linux profile") + + def _create_profile_folder(self, folder=None): + folder = folder or temp_folder(path_with_spaces=False) + save_files(folder, {"settings.yml": settings_yml, + "remotes.txt": remotes, + "profiles/linux": linux_profile, + "profiles/windows": win_profile, + "config/conan.conf": conan_conf, + "pylintrc": "#Custom pylint"}) + return folder + + def _create_zip(self, zippath=None): + folder = self._create_profile_folder() + zippath = zippath or os.path.join(folder, "myconfig.zip") + zipdir(folder, zippath) + return zippath + + def _check(self, install_path): + settings_path = self.client.client_cache.settings_path + self.assertEqual(load(settings_path).splitlines(), settings_yml.splitlines()) + registry_path = self.client.client_cache.registry + registry = RemoteRegistry(registry_path, TestBufferConanOutput()) + self.assertEqual(registry.remotes, + [Remote("myrepo1", "https://myrepourl.net", False), + Remote("my-repo-2", "https://myrepo2.com", True), + ]) + self.assertEqual(registry.refs, {"MyPkg/0.1@user/channel": "my-repo-2"}) + self.assertEqual(sorted(os.listdir(self.client.client_cache.profiles_path)), + sorted(["default", "linux", "windows"])) + self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "linux")).splitlines(), + linux_profile.splitlines()) + self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "windows")).splitlines(), + win_profile.splitlines()) + conan_conf = ConanClientConfigParser(self.client.client_cache.conan_conf_path) + self.assertEqual(conan_conf.get_item("log.run_to_output"), "False") + self.assertEqual(conan_conf.get_item("log.run_to_file"), "False") + self.assertEqual(conan_conf.get_item("log.level"), "10") + self.assertEqual(conan_conf.get_item("general.compression_level"), "6") + self.assertEqual(conan_conf.get_item("general.sysrequires_sudo"), "True") + self.assertEqual(conan_conf.get_item("general.cpu_count"), "1") + self.assertEqual(conan_conf.get_item("general.config_install"), install_path) + self.assertEqual(conan_conf.get_item("proxies.no_proxy"), "mylocalhost") + self.assertEqual(conan_conf.get_item("proxies.https"), "None") + self.assertEqual(conan_conf.get_item("proxies.http"), "http://user:pass@10.10.1.10:3128/") + self.assertEqual("#Custom pylint", + load(os.path.join(self.client.client_cache.conan_folder, "pylintrc"))) + + def install_file_test(self): + """ should install from a file in current dir + """ + zippath = self._create_zip() + self.client.run('config install "%s"' % zippath) + self._check(zippath) + + def test_without_profile_folder(self): + shutil.rmtree(self.client.client_cache.profiles_path) + zippath = self._create_zip() + self.client.run('config install "%s"' % zippath) + self.assertEqual(sorted(os.listdir(self.client.client_cache.profiles_path)), + sorted(["linux", "windows"])) + self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "linux")).splitlines(), + linux_profile.splitlines()) + + def install_url_test(self): + """ should install from a URL + """ + + def my_download(obj, url, filename, **kwargs): # @UnusedVariable + self._create_zip(filename) + + with patch.object(Downloader, 'download', new=my_download): + self.client.run("config install http://myfakeurl.com/myconf.zip") + self._check("http://myfakeurl.com/myconf.zip") + + # repeat the process to check + self.client.run("config install http://myfakeurl.com/myconf.zip") + self._check("http://myfakeurl.com/myconf.zip") + + def install_repo_test(self): + """ should install from a git repo + """ + + folder = self._create_profile_folder() + with tools.chdir(folder): + self.client.runner('git init .') + self.client.runner('git add .') + self.client.runner('git config user.name myname') + self.client.runner('git config user.email myname@mycompany.com') + self.client.runner('git commit -m "mymsg"') + + self.client.run('config install "%s/.git"' % folder) + self._check("%s/.git" % folder) + + def reinstall_test(self): + """ should use configured URL in conan.conf + """ + zippath = self._create_zip() + self.client.run('config set general.config_install="%s"' % zippath) + self.client.run("config install") + self._check(zippath) + + def reinstall_error_test(self): + """ should use configured URL in conan.conf + """ + error = self.client.run("config install", ignore_error=True) + self.assertTrue(error) + self.assertIn("Called config install without arguments", self.client.out) diff --git a/conans/test/command/config_test.py b/conans/test/command/config_test.py index e0bd1ded2d9..2d6d64c77aa 100644 --- a/conans/test/command/config_test.py +++ b/conans/test/command/config_test.py @@ -1,10 +1,7 @@ -from conans.test.utils.tools import TestClient import unittest -from conans.util.files import save, load -from conans.client.conf import default_client_conf -from conans import tools -from conans.test.utils.test_files import temp_folder -import os + +from conans.util.files import load +from conans.test.utils.tools import TestClient class ConfigTest(unittest.TestCase): diff --git a/conans/test/command/create_test.py b/conans/test/command/create_test.py index c82ab682f88..af041a01fd2 100644 --- a/conans/test/command/create_test.py +++ b/conans/test/command/create_test.py @@ -164,7 +164,7 @@ def build(self): [build_requires] BuildRequire/0.1@conan/stable ''', -"test_package/conanfile.py": """from conans import ConanFile + "test_package/conanfile.py": """from conans import ConanFile import os class MyTest(ConanFile): diff --git a/conans/test/command/info_test.py b/conans/test/command/info_test.py index c6ecd1d0fc0..ad20a8ad9c4 100644 --- a/conans/test/command/info_test.py +++ b/conans/test/command/info_test.py @@ -294,15 +294,11 @@ def build_order_test(self): def diamond_build_order_test(self): self.client = TestClient() self._create("LibA", "0.1") - self._create("Dev1", "0.1") - self._create("LibE", "0.1", deps_dev=["Dev1/0.1@lasote/stable"]) + self._create("LibE", "0.1") self._create("LibF", "0.1") - self._create("LibG", "0.1") - self._create("Dev2", "0.1", deps=["LibG/0.1@lasote/stable"]) self._create("LibB", "0.1", ["LibA/0.1@lasote/stable", "LibE/0.1@lasote/stable"]) - self._create("LibC", "0.1", ["LibA/0.1@lasote/stable", "LibF/0.1@lasote/stable"], - deps_dev=["Dev2/0.1@lasote/stable"]) + self._create("LibC", "0.1", ["LibA/0.1@lasote/stable", "LibF/0.1@lasote/stable"]) self._create("LibD", "0.1", ["LibB/0.1@lasote/stable", "LibC/0.1@lasote/stable"], export=False) @@ -321,22 +317,15 @@ def diamond_build_order_test(self): self.client.user_io.out) self.client.run("info -bo=Dev1/0.1@lasote/stable") self.assertEqual("\n", self.client.user_io.out) - self.client.run("info --scope=LibE:dev=True -bo=Dev1/0.1@lasote/stable") - self.assertIn("[Dev1/0.1@lasote/stable], [LibE/0.1@lasote/stable], " - "[LibB/0.1@lasote/stable]", self.client.user_io.out) self.client.run("info -bo=LibG/0.1@lasote/stable") self.assertEqual("\n", self.client.user_io.out) - self.client.run("info --scope=LibC:dev=True -bo=LibG/0.1@lasote/stable") - self.assertIn("[LibG/0.1@lasote/stable], [Dev2/0.1@lasote/stable], " - "[LibC/0.1@lasote/stable]", self.client.user_io.out) self.client.run("info --build_order=ALL") self.assertIn("[LibA/0.1@lasote/stable, LibE/0.1@lasote/stable, LibF/0.1@lasote/stable], " "[LibB/0.1@lasote/stable, LibC/0.1@lasote/stable]", self.client.user_io.out) - self.client.run("info --build_order=ALL --scope=ALL:dev=True") - self.assertIn("[Dev1/0.1@lasote/stable, LibG/0.1@lasote/stable], " - "[Dev2/0.1@lasote/stable, LibA/0.1@lasote/stable, LibE/0.1@lasote/stable, " + self.client.run("info --build_order=ALL") + self.assertIn("[LibA/0.1@lasote/stable, LibE/0.1@lasote/stable, " "LibF/0.1@lasote/stable], [LibB/0.1@lasote/stable, LibC/0.1@lasote/stable]", self.client.user_io.out) diff --git a/conans/test/command/install_subfolder_test.py b/conans/test/command/install_subfolder_test.py index 1a953aebb8a..e4145586439 100644 --- a/conans/test/command/install_subfolder_test.py +++ b/conans/test/command/install_subfolder_test.py @@ -66,6 +66,6 @@ def reuse_test(self): (1, h01, h11, h00, h10)]: self.client.current_folder = os.path.join(current_folder, "lang%dbuild" % lang) self.client.run("build ..") - self.assertIn("compiler=Visual Studio", self.client.user_io.out) - self.assertIn("language=%d" % lang, self.client.user_io.out) - self.assertNotIn("language=%d" % (not lang), self.client.user_io.out) + self.assertIn("compiler=Visual Studio", self.client.out) + self.assertIn("language=%d" % lang, self.client.out) + self.assertNotIn("language=%d" % (not lang), self.client.out) diff --git a/conans/test/command/profile_test.py b/conans/test/command/profile_test.py index 8daef183494..cdcf9b1dcc2 100644 --- a/conans/test/command/profile_test.py +++ b/conans/test/command/profile_test.py @@ -10,12 +10,12 @@ class ProfileTest(unittest.TestCase): def empty_test(self): - client = TestClient() + client = TestClient(default_profile=False) client.run("profile list") self.assertIn("No profiles defined", client.user_io.out) def list_test(self): - client = TestClient() + client = TestClient(default_profile=False) create_profile(client.client_cache.profiles_path, "profile3") create_profile(client.client_cache.profiles_path, "profile1") create_profile(client.client_cache.profiles_path, "profile2") @@ -24,7 +24,7 @@ def list_test(self): list(str(client.user_io.out).splitlines())) def show_test(self): - client = TestClient() + client = TestClient(default_profile=False) create_profile(client.client_cache.profiles_path, "profile1", settings={"os": "Windows"}, options=[("MyOption", "32")]) create_profile(client.client_cache.profiles_path, "profile2", scopes={"test": True}) @@ -41,7 +41,7 @@ def show_test(self): self.assertIn(" CXX=/path/tomy/g++_build", client.user_io.out) self.assertIn(" package:VAR=value", client.user_io.out) - def profile_update_test(self): + def profile_update_and_get_test(self): client = TestClient() client.run("profile new ./MyProfile --detect") pr_path = os.path.join(client.current_folder, "MyProfile") @@ -50,18 +50,33 @@ def profile_update_test(self): self.assertIn("os=FakeOS", load(pr_path)) self.assertNotIn("os=Linux", load(pr_path)) + client.run("profile get settings.os ./MyProfile") + self.assertEquals(client.out, "FakeOS\n") + client.run("profile update settings.compiler.version=88 ./MyProfile") self.assertIn("compiler.version=88", load(pr_path)) + client.run("profile get settings.compiler.version ./MyProfile") + self.assertEquals(client.out, "88\n") + client.run("profile update options.MyOption=23 ./MyProfile") self.assertIn("[options]\nMyOption=23", load(pr_path)) + client.run("profile get options.MyOption ./MyProfile") + self.assertEquals(client.out, "23\n") + client.run("profile update options.Package:MyOption=23 ./MyProfile") self.assertIn("Package:MyOption=23", load(pr_path)) + client.run("profile get options.Package:MyOption ./MyProfile") + self.assertEquals(client.out, "23\n") + client.run("profile update options.Package:OtherOption=23 ./MyProfile") self.assertIn("Package:OtherOption=23", load(pr_path)) + client.run("profile get options.Package:OtherOption ./MyProfile") + self.assertEquals(client.out, "23\n") + client.run("profile update scopes.Package:OneScope=True ./MyProfile") self.assertIn("[scopes]\nPackage:OneScope=True", load(pr_path)) @@ -72,6 +87,9 @@ def profile_update_test(self): client.run("profile update env.OneMyEnv=MYVALUe ./MyProfile") self.assertIn("[env]\nOneMyEnv=MYVALUe", load(pr_path)) + client.run("profile get env.OneMyEnv ./MyProfile") + self.assertEquals(client.out, "MYVALUe\n") + # Now try the remove client.run("profile remove settings.os ./MyProfile") diff --git a/conans/test/command/source_test.py b/conans/test/command/source_test.py index 84da89dfe3c..2084897a7c1 100644 --- a/conans/test/command/source_test.py +++ b/conans/test/command/source_test.py @@ -10,12 +10,14 @@ class SourceTest(unittest.TestCase): def basic_source_test(self): conanfile = ''' from conans import ConanFile +import os class ConanLib(ConanFile): name = "Hello" version = "0.1" def source(self): + assert(os.listdir(".") == []) # Not conanfile copied, clean source self.output.info("Running source!") ''' client = TestClient() @@ -36,6 +38,27 @@ def source(self): self.assertIn("Hello/0.1@lasote/stable: Configuring sources", client.user_io.out) self.assertIn("Hello/0.1@lasote/stable: Running source!", client.user_io.out) + def source_local_cwd_test(self): + conanfile = ''' +import os +from conans import ConanFile + +class ConanLib(ConanFile): + name = "Hello" + version = "0.1" + + def source(self): + self.output.info("Running source!") + self.output.info("cwd=>%s" % os.getcwd()) +''' + client = TestClient() + client.save({CONANFILE: conanfile}) + subdir = os.path.join(client.current_folder, "subdir") + os.mkdir(subdir) + client.run("source .. --cwd subdir") + self.assertIn("PROJECT: Configuring sources", client.user_io.out) + self.assertIn("PROJECT: cwd=>%s" % subdir, client.user_io.out) + def local_source_test(self): conanfile = ''' from conans import ConanFile @@ -60,6 +83,5 @@ def source(self): client.save({CONANFILE: conanfile.replace("err", "")}) client.run("source .") self.assertIn("PROJECT: Configuring sources in", client.user_io.out) - self.assertIn("PROJECT: WARN: Your previous source command failed", client.user_io.out) self.assertIn("PROJECT: Running source!", client.user_io.out) self.assertEqual("Hello World", load(os.path.join(client.current_folder, "file1.txt"))) diff --git a/conans/test/conan_api/__init__.py b/conans/test/conan_api/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/test/conan_api/manifests_arguments_test.py b/conans/test/conan_api/manifests_arguments_test.py new file mode 100644 index 00000000000..0c315c7dc2e --- /dev/null +++ b/conans/test/conan_api/manifests_arguments_test.py @@ -0,0 +1,71 @@ +from conans.client.conan_api import _parse_manifests_arguments, ConanException, default_manifest_folder, prepare_cwd +import unittest +from nose_parameterized.parameterized import parameterized + + +class ArgumentsTest(unittest.TestCase): + @parameterized.expand([ + (dict(verify=default_manifest_folder, + manifests=default_manifest_folder, + manifests_interactive=default_manifest_folder),), + (dict(verify=None, + manifests=default_manifest_folder, + manifests_interactive=default_manifest_folder),), + (dict(verify=default_manifest_folder, + manifests=None, + manifests_interactive=default_manifest_folder),), + (dict(verify=default_manifest_folder, + manifests=default_manifest_folder, + manifests_interactive=None),), + (dict(verify=default_manifest_folder, + manifests=None, + manifests_interactive=None),), + ]) + def test_manifest_arguments_conflicting(self, arguments): + with self.assertRaises(ConanException): + _parse_manifests_arguments(cwd=None, **arguments) + + def test_manifests_arguments_verify(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=default_manifest_folder, + manifests=None, + manifests_interactive=None, + cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIn(cwd, manifest_folder) + self.assertFalse(manifest_interactive) + self.assertTrue(manifest_verify) + + def test_manifests_arguments_manifests_interactive(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=None, + manifests=None, + manifests_interactive=default_manifest_folder, + cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIn(cwd, manifest_folder) + self.assertTrue(manifest_interactive) + self.assertFalse(manifest_verify) + + def test_manifests_arguments_manifests(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=None, + manifests=default_manifest_folder, + manifests_interactive=None, + cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIn(cwd, manifest_folder) + self.assertFalse(manifest_interactive) + self.assertFalse(manifest_verify) + + def test_manifests_arguments_no_manifests(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=None, manifests=None, manifests_interactive=None, cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIsNone(manifest_folder) + self.assertFalse(manifest_interactive) + self.assertFalse(manifest_verify) diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 30a0f16b3e6..564d9c50982 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -1,7 +1,6 @@ import os import shutil import sys -import tempfile import unittest import platform @@ -12,17 +11,61 @@ from conans.model.settings import Settings from conans.client.conf import default_settings_yml from conans.client.cmake import CMake +from conans.test.utils.tools import TestBufferConanOutput from conans.tools import cpu_count from conans.util.files import save +from conans.test.utils.test_files import temp_folder class CMakeTest(unittest.TestCase): def setUp(self): - self.tempdir = tempfile.mkdtemp() + self.tempdir = temp_folder(path_with_spaces=False) def tearDown(self): shutil.rmtree(self.tempdir) + def build_type_ovewrite_test(self): + settings = Settings.loads(default_settings_yml) + settings.os = "Linux" + settings.compiler = "gcc" + settings.compiler.version = "6.3" + settings.arch = "x86" + settings.build_type = "Release" + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + cmake.build_type = "Debug" + self.assertIn('WARN: Set CMake build type "Debug" is different than the ' + 'settings build_type "Release"', conan_file.output) + self.assertEquals(cmake.build_type, "Debug") + self.assertIn('-DCMAKE_BUILD_TYPE="Debug"', cmake.command_line) + + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + self.assertNotIn('WARN: Set CMake build type ', conan_file.output) + self.assertEquals(cmake.build_type, "Release") + + # Now with visual, (multiconfig) + settings = Settings.loads(default_settings_yml) + settings.os = "Windows" + settings.compiler = "Visual Studio" + settings.compiler.version = "15" + settings.arch = "x86" + settings.build_type = "Release" + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + cmake.build_type = "Debug" + self.assertIn('WARN: Set CMake build type "Debug" is different than the ' + 'settings build_type "Release"', conan_file.output) + self.assertEquals(cmake.build_type, "Debug") + self.assertNotIn('-DCMAKE_BUILD_TYPE="Debug"', cmake.command_line) + self.assertIn("--config Debug", cmake.build_config) + cmake = CMake(conan_file) + cmake.build_type = "Release" + self.assertIn("--config Release", cmake.build_config) + def loads_default_test(self): settings = Settings.loads(default_settings_yml) settings.os = "Windows" @@ -420,7 +463,7 @@ def __init__(self, shared=None): self.source_folder = self.build_folder = "." self.settings = None self.deps_cpp_info = namedtuple("deps_cpp_info", "sysroot")("/path/to/sysroot") - self.output = namedtuple("output", "warn")(lambda x: x) + self.output = TestBufferConanOutput() if shared is not None: self.options = namedtuple("options", "shared")(shared) diff --git a/conans/test/functional/compile_helpers_test.py b/conans/test/functional/compile_helpers_test.py index 37876e9aed2..fc93ee8656c 100644 --- a/conans/test/functional/compile_helpers_test.py +++ b/conans/test/functional/compile_helpers_test.py @@ -419,7 +419,7 @@ def append_variables_test(self): win_settings = MockSettings("Release", os="Windows", arch="x86", compiler_name="Visual Studio", libcxx=None, - version="12") + version="14") env = ConfigureEnvironment(MockConanfile(win_settings)) command = "%s && SET" % env.command_line runner(command, output=output) diff --git a/conans/test/functional/conan_settings_preprocessor_test.py b/conans/test/functional/conan_settings_preprocessor_test.py index 70bf4bd0dc6..f36452655f1 100644 --- a/conans/test/functional/conan_settings_preprocessor_test.py +++ b/conans/test/functional/conan_settings_preprocessor_test.py @@ -8,7 +8,7 @@ class ConanSettingsPreprocessorTest(unittest.TestCase): def setUp(self): - self.client = TestClient() + self.client = TestClient(default_profile=False) self.conanfile = ''' from conans import ConanFile @@ -52,4 +52,4 @@ def test_runtime_not_present_ok(self): # Now install, the preprocessor shouldn't fail nor do anything self.client.run("install Hello0/0.1@lasote/channel --build missing") self.assertNotIn("Setting 'compiler.runtime' not declared, automatically", - self.client.user_io.out) \ No newline at end of file + self.client.user_io.out) diff --git a/conans/test/functional/in_local_cache_test.py b/conans/test/functional/in_local_cache_test.py new file mode 100644 index 00000000000..61971d51790 --- /dev/null +++ b/conans/test/functional/in_local_cache_test.py @@ -0,0 +1,69 @@ +import os +import unittest +from conans.test.utils.tools import TestClient +from conans.paths import CONANFILE + + +conanfile = """ +from conans import ConanFile, tools + +class AConan(ConanFile): + name = "Hello0" + version = "0.1" + + def build(self): + self.output.warn("build() IN LOCAL CACHE=> %s" % str(self.in_local_cache)) + + def package(self): + self.output.warn("package() IN LOCAL CACHE=> %s" % str(self.in_local_cache)) + +""" + + +class InLocalCacheTest(unittest.TestCase): + + def test_in_local_cache_flag(self): + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("export lasote/stable") + client.run("install Hello0/0.1@lasote/stable --build missing") + self.assertIn("build() IN LOCAL CACHE=> True", client.user_io.out) + self.assertIn("package() IN LOCAL CACHE=> True", client.user_io.out) + + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("install .") + client.run("build") + self.assertIn("build() IN LOCAL CACHE=> False", client.user_io.out) + + pack_folder = os.path.join(client.current_folder, "package") + os.mkdir(pack_folder) + client.current_folder = pack_folder + client.run("package .. --build_folder ..") + self.assertIn("package() IN LOCAL CACHE=> False", client.user_io.out) + + # Confirm that we have the flag depending on the recipe too + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("export lasote/stable") + conanfile_reuse = """ +from conans import ConanFile, tools + +class OtherConan(ConanFile): + name = "Hello1" + version = "0.1" + requires = "Hello0/0.1@lasote/stable" + + def build(self): + pass +""" + client.save({CONANFILE: conanfile_reuse}, clean_first=True) + client.run("install . --build") + self.assertIn("build() IN LOCAL CACHE=> True", client.user_io.out) + self.assertIn("package() IN LOCAL CACHE=> True", client.user_io.out) + client.run("export lasote/stable") + client.run("install Hello1/0.1@lasote/stable --build") + self.assertIn("build() IN LOCAL CACHE=> True", client.user_io.out) + self.assertIn("package() IN LOCAL CACHE=> True", client.user_io.out) + + diff --git a/conans/test/functional/path_limit_test.py b/conans/test/functional/path_limit_test.py index 3fa19308690..a7eab70fba9 100644 --- a/conans/test/functional/path_limit_test.py +++ b/conans/test/functional/path_limit_test.py @@ -196,6 +196,39 @@ def basic_test(self): self.assertFalse(os.path.exists(link_build)) self.assertFalse(os.path.exists(link_package)) + def basic_disabled_test(self): + client = TestClient() + base = ''' +from conans import ConanFile + +class ConanLib(ConanFile): + short_paths = True +''' + files = {"conanfile.py": base} + client.save(files) + client.client_cache.conan_config.set_item("general.user_home_short", "None") + + client.run("create lib/0.1@user/channel") + package_ref = PackageReference.loads("lib/0.1@user/channel:" + "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") + client.run("search") + self.assertIn("lib/0.1@user/channel", client.user_io.out) + client.run("search lib/0.1@user/channel") + self.assertIn("Package_ID: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", client.user_io.out) + + conan_ref = ConanFileReference.loads("lib/0.1@user/channel") + source_folder = client.client_cache.source(conan_ref) + link_source = os.path.join(source_folder, ".conan_link") + self.assertFalse(os.path.exists(link_source)) + + build_folder = client.client_cache.build(package_ref) + link_build = os.path.join(build_folder, ".conan_link") + self.assertFalse(os.path.exists(link_build)) + + package_folder = client.client_cache.package(package_ref) + link_package = os.path.join(package_folder, ".conan_link") + self.assertFalse(os.path.exists(link_package)) + def failure_test(self): base = ''' diff --git a/conans/test/functional/proxies_conf_test.py b/conans/test/functional/proxies_conf_test.py new file mode 100644 index 00000000000..8e0b3e89d43 --- /dev/null +++ b/conans/test/functional/proxies_conf_test.py @@ -0,0 +1,29 @@ +import unittest +import os + +from conans.test.utils.tools import TestClient +from conans.util.files import save +from conans.client.conan_api import get_basic_requester + + +class ProxiesConfTest(unittest.TestCase): + def setUp(self): + self.old_env = dict(os.environ) + + def tearDown(self): + os.environ.clear() + os.environ.update(self.old_env) + + def test_requester(self): + client = TestClient(default_profile=False) + conf = """ +[proxies] +https=None +no_proxy=http://someurl,http://otherurl.com +http=http:/conan.url + """ + save(client.client_cache.conan_conf_path, conf) + requester = get_basic_requester(client.client_cache) + self.assertEqual(requester.proxies, {"https": None, + "http": "http:/conan.url"}) + self.assertEqual(os.environ["NO_PROXY"], "http://someurl,http://otherurl.com") diff --git a/conans/test/generators/generators_test.py b/conans/test/generators/generators_test.py index abf36a6d1ac..eab8e62561c 100644 --- a/conans/test/generators/generators_test.py +++ b/conans/test/generators/generators_test.py @@ -16,6 +16,7 @@ def test_base(self): scons txt visual_studio +visual_studio_legacy xcode ycm ''' @@ -26,7 +27,8 @@ def test_base(self): self.assertEqual(sorted(['conanfile.txt', 'conaninfo.txt', 'conanbuildinfo.cmake', 'conanbuildinfo.gcc', 'conanbuildinfo.qbs', 'conanbuildinfo.pri', 'SConscript_conan', 'conanbuildinfo.txt', 'conanbuildinfo.props', - 'conanbuildinfo.xcconfig', '.ycm_extra_conf.py']), + 'conanbuildinfo.vsprops', 'conanbuildinfo.xcconfig', + '.ycm_extra_conf.py']), sorted(os.listdir(client.current_folder))) def test_qmake(self): diff --git a/conans/test/generators/visual_studio_legacy_test.py b/conans/test/generators/visual_studio_legacy_test.py new file mode 100644 index 00000000000..c12dc7e0c0a --- /dev/null +++ b/conans/test/generators/visual_studio_legacy_test.py @@ -0,0 +1,40 @@ +import unittest +import xml.etree.ElementTree + +from conans.client.generators import VisualStudioLegacyGenerator + +from conans.model.settings import Settings +from conans.model.conan_file import ConanFile +from conans.model.build_info import CppInfo +from conans.model.ref import ConanFileReference +from conans.test.utils.test_files import temp_folder +import os + + +class VisualStudioLegacyGeneratorTest(unittest.TestCase): + + def valid_xml_test(self): + conanfile = ConanFile(None, None, Settings({}), None) + ref = ConanFileReference.loads("MyPkg/0.1@user/testing") + folder1 = temp_folder() + folder1 = folder1.replace("\\", "/") + os.makedirs(os.path.join(folder1, "include")) + os.makedirs(os.path.join(folder1, "lib")) + cpp_info = CppInfo(folder1) + conanfile.deps_cpp_info.update(cpp_info, ref.name) + ref = ConanFileReference.loads("My.Fancy-Pkg_2/0.1@user/testing") + folder2 = temp_folder() + folder2 = folder2.replace("\\", "/") + os.makedirs(os.path.join(folder2, "include")) + os.makedirs(os.path.join(folder2, "lib")) + cpp_info = CppInfo(folder2) + conanfile.deps_cpp_info.update(cpp_info, ref.name) + generator = VisualStudioLegacyGenerator(conanfile) + + content = generator.content + xml.etree.ElementTree.fromstring(content) + + self.assertIn('AdditionalIncludeDirectories=""%s/include";"%s/include";"' + % (folder1, folder2), content) + self.assertIn('AdditionalLibraryDirectories=""%s/lib";"%s/lib";"' + % (folder1, folder2), content) diff --git a/conans/test/generators/visual_studio_test.py b/conans/test/generators/visual_studio_test.py index 960a3ee3351..0f43200ec8e 100644 --- a/conans/test/generators/visual_studio_test.py +++ b/conans/test/generators/visual_studio_test.py @@ -1,4 +1,3 @@ -import re import unittest import xml.etree.ElementTree @@ -6,13 +5,13 @@ from conans.model.settings import Settings from conans.model.conan_file import ConanFile -from conans.client.generators.cmake import CMakeGenerator from conans.model.build_info import CppInfo from conans.model.ref import ConanFileReference class VisualStudioGeneratorTest(unittest.TestCase): - def _createInfo(self): + + def valid_xml_test(self): conanfile = ConanFile(None, None, Settings({}), None) ref = ConanFileReference.loads("MyPkg/0.1@user/testing") cpp_info = CppInfo("dummy_root_folder1") @@ -21,19 +20,10 @@ def _createInfo(self): cpp_info = CppInfo("dummy_root_folder2") conanfile.deps_cpp_info.update(cpp_info, ref.name) generator = VisualStudioGenerator(conanfile) - return generator.content - def valid_xml_test(self): - data = self._createInfo() - try: - xml.etree.ElementTree.fromstring(data) - except xml.etree.ElementTree.ParseError as err: - self.fail("Visual studio generated code is not valid! Error %s:\n%s " % (str(err), data)) + content = generator.content + xml.etree.ElementTree.fromstring(content) - - def variables_setup_test(self): - content = self._createInfo() self.assertIn('', content) self.assertIn("dummy_root_folder1", content) self.assertIn("dummy_root_folder2", content) - diff --git a/conans/test/integration/basic_build_test.py b/conans/test/integration/basic_build_test.py index b4aec86f341..90ab4b70073 100644 --- a/conans/test/integration/basic_build_test.py +++ b/conans/test/integration/basic_build_test.py @@ -37,23 +37,17 @@ def _build(self, cmd, static, pure_c, use_cmake, lang): self.assertFalse(conan_info.full_options.static) def build_cmake_test(self): - for pure_c in (False, True): - for cmd, lang, static in [("install", 0, True), - ("install -o language=1", 1, True), - ("install -o language=1 -o static=False", 1, False), - ("install -o static=False", 0, False)]: - self._build(cmd, static, pure_c, use_cmake=True, lang=lang) + for cmd, lang, static, pure_c in [("install", 0, True, True), + ("install -o language=1 -o static=False", 1, False, False)]: + self._build(cmd, static, pure_c, use_cmake=True, lang=lang) def build_default_test(self): "build default (gcc in nix, VS in win)" if platform.system() == "SunOS": return # If is using sun-cc the gcc generator doesn't work - for pure_c in (False, True): - for cmd, lang, static in [("install -g txt", 0, True), - ("install -o language=1 -g txt", 1, True), - ("install -o language=1 -o static=False -g txt", 1, False), - ("install -o static=False -g txt", 0, False)]: - self._build(cmd, static, pure_c, use_cmake=False, lang=lang) + for cmd, lang, static, pure_c in [("install -g txt", 0, True, True), + ("install -o language=1 -o static=False -g txt", 1, False, False)]: + self._build(cmd, static, pure_c, use_cmake=False, lang=lang) def build_mingw_test(self): if platform.system() != "Windows": @@ -63,9 +57,6 @@ def build_mingw_test(self): logger.error("This platform does not support G++ command") return install = "install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9" - for pure_c in (False, True): - for cmd, lang, static in [(install, 0, True), - (install + " -o language=1", 1, True), - (install + " -o language=1 -o static=False", 1, False), - (install + " -o static=False", 0, False)]: - self._build(cmd, static, pure_c, use_cmake=False, lang=lang) + for cmd, lang, static, pure_c in [(install, 0, True, True), + (install + " -o language=1 -o static=False", 1, False, False)]: + self._build(cmd, static, pure_c, use_cmake=False, lang=lang) diff --git a/conans/test/integration/build_environment_test.py b/conans/test/integration/build_environment_test.py index b108d8a7647..eeae42e1fc9 100644 --- a/conans/test/integration/build_environment_test.py +++ b/conans/test/integration/build_environment_test.py @@ -2,6 +2,8 @@ import platform import unittest +from nose.plugins.attrib import attr + from conans.model.ref import ConanFileReference from conans.paths import CONANFILE from conans.test.utils.tools import TestClient @@ -63,6 +65,7 @@ def package_info(self): class BuildEnvironmenTest(unittest.TestCase): + @attr("mingw") def test_gcc_and_environment(self): if platform.system() == "SunOS": return # If is using sun-cc the gcc generator doesn't work diff --git a/conans/test/integration/build_id_test.py b/conans/test/integration/build_id_test.py index 8f2536ee063..85153020e61 100644 --- a/conans/test/integration/build_id_test.py +++ b/conans/test/integration/build_id_test.py @@ -1,10 +1,11 @@ -import unittest -from conans.test.utils.tools import TestClient import os -from conans.util.files import load -from conans.model.ref import PackageReference, ConanFileReference +import unittest + from nose_parameterized.parameterized import parameterized +from conans.model.ref import PackageReference, ConanFileReference +from conans.test.utils.tools import TestClient +from conans.util.files import load conanfile = """from conans import ConanFile from conans.util.files import save @@ -56,7 +57,6 @@ def imports(self): class BuildIdTest(unittest.TestCase): - def _check_conaninfo(self, client): # Check that conaninfo is correct ref_debug = PackageReference.loads("Pkg/0.1@user/channel:" diff --git a/conans/test/integration/cmake_multi_test.py b/conans/test/integration/cmake_multi_test.py index 152c4d60585..a85668f4660 100644 --- a/conans/test/integration/cmake_multi_test.py +++ b/conans/test/integration/cmake_multi_test.py @@ -130,6 +130,7 @@ def package_files(name, deps=None): @attr("slow") class CMakeMultiTest(unittest.TestCase): + @attr("mingw") def cmake_multi_find_test(self): if platform.system() not in ["Windows", "Linux"]: return diff --git a/conans/test/integration/conan_env_test.py b/conans/test/integration/conan_env_test.py index d907ea0ddad..5031a6458e7 100644 --- a/conans/test/integration/conan_env_test.py +++ b/conans/test/integration/conan_env_test.py @@ -264,6 +264,8 @@ def test_run_env(self): client = TestClient() conanfile = ''' from conans import ConanFile +from conans.tools import mkdir +import os class HelloConan(ConanFile): name = "Hello" @@ -271,6 +273,8 @@ class HelloConan(ConanFile): build_policy = "missing" def package_info(self): + mkdir(os.path.join(self.package_folder, "bin2")) + mkdir(os.path.join(self.package_folder, "lib2")) self.cpp_info.bindirs.append("bin2") self.cpp_info.libdirs.append("lib2") diff --git a/conans/test/integration/conan_scopes_test.py b/conans/test/integration/conan_scopes_test.py index 835f1640ff7..cc99f97ae57 100644 --- a/conans/test/integration/conan_scopes_test.py +++ b/conans/test/integration/conan_scopes_test.py @@ -33,7 +33,7 @@ class HelloConan(ConanFile): version = "0.1" def config(self): if self.scope.other: - self.requires("Hello/0.1@lasote/stable", dev=True) + self.requires("Hello/0.1@lasote/stable") ''' files["conanfile.py"] = conanfile client.save(files, clean_first=True) @@ -215,49 +215,3 @@ def build(self): self.assertIn("WARN: CONFIG_CONSUMER OTHER", client.user_io.out) self.assertNotIn("WARN: BUILD_CONSUMER DEV", client.user_io.out) self.assertNotIn("WARN: BUILD_CONSUMER OTHER", client.user_io.out) - - def conan_dev_requires_test(self): - client = TestClient() - conanfile = ''' -from conans import ConanFile - -class HelloConan(ConanFile): - name = "Base" - version = "0.1" -''' - files = {} - files["conanfile.py"] = conanfile - client.save(files) - client.run("export lasote/stable") - conanfile = ''' -from conans import ConanFile - -class HelloConan(ConanFile): - dev_requires = "Base/0.1@lasote/stable" - name = "Hello" - version = "0.1" -''' - files = {} - files["conanfile.py"] = conanfile - client.save(files) - client.run("export lasote/stable") - conanfile = ''' -from conans import ConanFile - -class HelloConan(ConanFile): - dev_requires = "Hello/0.1@lasote/stable" - ''' - files["conanfile.py"] = conanfile - client.save(files, clean_first=True) - - client.run("install --build") - self.assertIn("Hello/0.1@lasote/stable:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - client.user_io.out) - self.assertNotIn("Base/0.1@lasote/stable", client.user_io.out) - client.run("install --build -sc dev=False") - self.assertNotIn("Hello/0.1@lasote/stable", client.user_io.out) - self.assertNotIn("Base/0.1@lasote/stable", client.user_io.out) - client.run("install --build -sc dev=True -sc Hello:dev=True") - self.assertIn("Hello/0.1@lasote/stable:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - client.user_io.out) - self.assertIn("Base/0.1@lasote/stable", client.user_io.out) diff --git a/conans/test/integration/exception_printing_test.py b/conans/test/integration/exception_printing_test.py index 3c3ea4f9c40..62ad9bd7433 100644 --- a/conans/test/integration/exception_printing_test.py +++ b/conans/test/integration/exception_printing_test.py @@ -52,7 +52,6 @@ def _aux_method(self): def setUp(self): self.client = TestClient() - def _call_install(self, conanfile): self.client.save({CONANFILE: conanfile}, clean_first=True) self.client.run("export lasote/stable") @@ -78,10 +77,13 @@ def _get_conanfile_for(self, method_name): config_options_contents=throw if method_name == "config_options" else "pass") return cf - def _test_fail_line_aux(self, conanfile, numline, method_name): + def _test_fail_line_aux(self, conanfile, main_line, numline, method_name): self._call_install(conanfile) - self.assertIn("ExceptionsTest/0.1@lasote/stable: Error in %s() method, while calling '_aux_method', line %s" % (method_name, numline), + self.assertIn("ExceptionsTest/0.1@lasote/stable: Error in %s() method, line %s" % (method_name, main_line), + self.client.user_io.out) + self.assertIn("\nwhile calling '_aux_method', line %s" % numline, self.client.user_io.out) + self.assertIn("DRLException: Oh! an error!", self.client.user_io.out) def _get_conanfile_for_error_in_other_method(self, method_name): @@ -107,13 +109,20 @@ def test_all_methods(self): self._test_fail_line(self._get_conanfile_for(method), line, method) def test_aux_method(self): - for method, line in [("source", 41), ("build", 41), - ("package", 41), ("package_info", 41), - ("configure", 41), ("build_id", 41), - ("package_id", 41), ("requirements", 41), - ("config_options", 41)]: - self._test_fail_line_aux(self._get_conanfile_for_error_in_other_method(method), line, method) - + for method, main_line, line in [("source", 14, 41), ("build", 17, 41), + ("package", 20, 41), ("package_info", 23, 41), + ("configure", 26, 41), ("build_id", 29, 41), + ("package_id", 32, 41), ("requirements", 35, 41), + ("config_options", 38, 41)]: + self._test_fail_line_aux(self._get_conanfile_for_error_in_other_method(method), + main_line, line, method) + + def test_complete_traceback(self): + with tools.environment_append({"CONAN_VERBOSE_TRACEBACK": "1"}): + self._call_install(self._get_conanfile_for_error_in_other_method("source")) + self.assertIn("ERROR: Traceback (most recent call last):", self.client.user_io.out) + self.assertIn('self._aux_method()', self.client.user_io.out) + self.assertIn("raise DRLException('Oh! an error!')", self.client.user_io.out) if __name__ == '__main__': diff --git a/conans/test/integration/export_sources_test.py b/conans/test/integration/export_sources_test.py index f90bbdcbd51..537944455cb 100644 --- a/conans/test/integration/export_sources_test.py +++ b/conans/test/integration/export_sources_test.py @@ -85,11 +85,11 @@ def setUp(self): def _check_source_folder(self, mode): """ Source folder MUST be always the same """ - expected_sources = ['conanfile.py', 'conanmanifest.txt', "hello.h"] + expected_sources = ["hello.h"] if mode == "both": expected_sources.append("data.txt") if mode == "nested" or mode == "overlap": - expected_sources = ['conanfile.py', 'conanmanifest.txt', "src/hello.h", "src/data.txt"] + expected_sources = ["src/hello.h", "src/data.txt"] expected_sources = sorted(expected_sources) self.assertEqual(scan_folder(self.source_folder), expected_sources) diff --git a/conans/test/integration/flat_requirements_test.py b/conans/test/integration/flat_requirements_test.py index c1e5c4463d0..98a652f9379 100644 --- a/conans/test/integration/flat_requirements_test.py +++ b/conans/test/integration/flat_requirements_test.py @@ -15,6 +15,14 @@ def setUp(self): self.conan_reference = ConanFileReference.loads("Hello0/0.1@lasote/stable") self.files = cpp_hello_conan_files("Hello0", "0.1", build=False) self.conan = TestClient() + package = """def package(self): + import os + os.mkdir(os.path.join(self.package_folder, "include")) + os.mkdir(os.path.join(self.package_folder, "lib")) + os.mkdir(os.path.join(self.package_folder, "bin")) +""" + self.files["conanfile.py"] = self.files["conanfile.py"].replace("def package(self):", + package) self.conan.save(self.files) self.conan.run("export lasote/stable") diff --git a/conans/test/integration/go_complete_test.py b/conans/test/integration/go_complete_test.py index cf61804d86e..8ad3561ef31 100644 --- a/conans/test/integration/go_complete_test.py +++ b/conans/test/integration/go_complete_test.py @@ -92,6 +92,7 @@ def reuse_test(self): 'reverse_test.go': reverse_test, 'reverse.txt': reverse, 'hello/helloreverse.txt': reverse} + files_without_conanfile = set(files.keys()) - set(["conanfile.py"]) self.client.save(files) self.client.run("export lasote/stable") self.client.run("install %s --build missing" % str(conan_reference)) @@ -99,7 +100,7 @@ def reuse_test(self): package_ids = self.client.paths.conan_packages(conan_reference) self.assertEquals(len(package_ids), 1) package_ref = PackageReference(conan_reference, package_ids[0]) - self._assert_package_exists(package_ref, self.client.paths, list(files.keys())) + self._assert_package_exists(package_ref, self.client.paths, files_without_conanfile) # Upload conans self.client.run("upload %s" % str(conan_reference)) @@ -113,7 +114,7 @@ def reuse_test(self): self.client.run("upload %s -p %s" % (str(conan_reference), str(package_ids[0]))) # Check library on server - self._assert_package_exists_in_server(package_ref, server_paths, list(files.keys())) + self._assert_package_exists_in_server(package_ref, server_paths, files_without_conanfile) # Now from other "computer" install the uploaded conans with same options (nothing) other_conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) @@ -122,7 +123,7 @@ def reuse_test(self): build_path = other_conan.paths.build(package_ref) self.assertFalse(os.path.exists(build_path)) # Lib should exist - self._assert_package_exists(package_ref, other_conan.paths, list(files.keys())) + self._assert_package_exists(package_ref, other_conan.paths, files_without_conanfile) reuse_conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) files = {'conanfile.py': reuse_conanfile, diff --git a/conans/test/integration/manifest_validation_test.py b/conans/test/integration/manifest_validation_test.py index 177535f330e..ddf69f6ef39 100644 --- a/conans/test/integration/manifest_validation_test.py +++ b/conans/test/integration/manifest_validation_test.py @@ -92,7 +92,7 @@ def _capture_verify_manifest(self, reference, remote="local cache", folder=""): # again should do nothing self.client.run("install %s --build missing --manifests %s" % (str(self.reference), folder)) - self.assertNotIn("manifest", self.client.user_io.out) + self.assertNotIn("Installed manifest", self.client.user_io.out) # now verify self.client.run("install %s --build missing --verify %s" % (str(self.reference), folder)) diff --git a/conans/test/integration/multi_build_test.py b/conans/test/integration/multi_build_test.py index 110eca40876..cc1c18eb1d1 100644 --- a/conans/test/integration/multi_build_test.py +++ b/conans/test/integration/multi_build_test.py @@ -23,7 +23,6 @@ def collect_libs_test(self): self.assertEquals(len(package_ids), 1) # Reuse them - conan_reference = ConanFileReference.loads("Hello1/0.2@lasote/stable") files3 = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], collect_libs=True) diff --git a/conans/test/integration/profile_test.py b/conans/test/integration/profile_test.py index d5f4a771661..d19cb719046 100644 --- a/conans/test/integration/profile_test.py +++ b/conans/test/integration/profile_test.py @@ -1,8 +1,5 @@ import unittest -from conans.client.profile_loader import _load_profile -from conans.model.env_info import EnvValues - from conans.test.utils.tools import TestClient from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.util.files import save, load diff --git a/conans/test/libcxx_setting_test.py b/conans/test/libcxx_setting_test.py index 2d5247a8f71..4c46f3e9853 100644 --- a/conans/test/libcxx_setting_test.py +++ b/conans/test/libcxx_setting_test.py @@ -12,8 +12,6 @@ class ConanFileToolsTest(ConanFile): name = "test" version = "1.9" settings = "os", "compiler", "arch", "build_type" - url = "1" - license = "2" export = ["CMakeLists.txt", "main.c"] generators = ["cmake"] @@ -21,14 +19,13 @@ def build(self): self.output.warn("Building...") cmake = CMake(self) self.output.warn(cmake.command_line) - command = cmake.command_line.replace('-G "Visual Studio 12 Win64"', "") - self.run('cmake . %s' % command) + self.run('cmake . %s' % cmake.command_line) self.run("cmake --build . %s" % cmake.build_config) def package(self): - self.copy("*", ".", ".") - + self.copy("*") ''' + cmakelists = '''PROJECT(conanzlib) set(CONAN_DISABLE_CHECK_COMPILER TRUE) cmake_minimum_required(VERSION 2.8) @@ -50,95 +47,85 @@ def nowintest(func): class LibcxxSettingTest(unittest.TestCase): - def setUp(self): - self.files = {"conanfile.py": file_content, "CMakeLists.txt": cmakelists} - @nowintest def test_declared_stdlib_and_passed(self): client = TestClient() - client.save(self.files) + client.save({"conanfile.py": file_content, + "CMakeLists.txt": cmakelists}) client.run("export lasote/testing") if platform.system() == "SunOS": - client.run('install -s compiler=sun-cc -s compiler.libcxx=libCstd', ignore_error=False) + client.run('install -s compiler=sun-cc -s compiler.libcxx=libCstd') client.run('build') - self.assertIn("-library=Cstd", str(client.user_io.out)) + self.assertIn("-library=Cstd", client.out) - client.run('install -s compiler=sun-cc -s compiler.libcxx=libstdcxx', ignore_error=False) + client.run('install -s compiler=sun-cc -s compiler.libcxx=libstdcxx') client.run('build') - self.assertIn("-library=stdcxx4", str(client.user_io.out)) + self.assertIn("-library=stdcxx4", client.out) - client.run('install -s compiler=sun-cc -s compiler.libcxx=libstlport', ignore_error=False) + client.run('install -s compiler=sun-cc -s compiler.libcxx=libstlport') client.run('build') - self.assertIn("-library=stlport4", str(client.user_io.out)) + self.assertIn("-library=stlport4", client.out) else: - client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++ ', ignore_error=False) + client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++ ') client.run('build') - self.assertIn("-stdlib=libstdc++", str(client.user_io.out)) - self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=0", str(client.user_io.out)) + self.assertIn("-stdlib=libstdc++", client.out) + self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=0", client.out) - client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++11', ignore_error=False) + client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++11') client.run('build') - self.assertIn("-stdlib=libstdc++", str(client.user_io.out)) - self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=1", str(client.user_io.out)) + self.assertIn("-stdlib=libstdc++", client.out) + self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=1", client.out) - client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libc++', ignore_error=False) + client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libc++') client.run('build') - self.assertIn("-stdlib=libc++", str(client.user_io.out)) - self.assertNotIn("Found Define: _GLIBCXX_USE_CXX11", str(client.user_io.out)) + self.assertIn("-stdlib=libc++", client.out) + self.assertNotIn("Found Define: _GLIBCXX_USE_CXX11", client.out) def test_C_only(self): - config = ''' - def config(self): + conanfile = """from conans import ConanFile + +class ConanFileToolsTest(ConanFile): + name = "test" + version = "1.9" + settings = "os", "compiler", "arch", "build_type" + + def configure(self): del self.settings.compiler.libcxx # C package only -''' - self.files["conanfile.py"] = self.files["conanfile.py"].replace('["cmake"]', - '["cmake"]\n %s' % config) + """ - self.files["conanfile.py"] = self.files["conanfile.py"].replace("def build", "def nobuild") client = TestClient() - client.save(self.files) - client.run("export lasote/testing") - client.run("install") + client.save({"conanfile.py": conanfile}) # Also check that it not fails the config method with Visual Studio, because of the lack of libcxx - client.run('install -s compiler="Visual Studio" -s compiler.version=12 -s compiler.runtime=MD', ignore_error=False) - self.assertIn("Generator cmake created conanbuildinfo.cmake", str(client.user_io.out)) + client.run('install -s compiler="Visual Studio" -s compiler.version=14') + self.assertIn("PROJECT: Generated conaninfo.txt", client.out) conaninfo = load(os.path.join(client.current_folder, "conaninfo.txt")) - self.assertNotIn("libcxx", conaninfo[:conaninfo.find("[full_settings]")]) - client.run('install test/1.9@lasote/testing -s compiler=gcc -s compiler.version=4.9 --build', ignore_error=False) + self.assertNotIn("libcxx", conaninfo) + client.run('install -s compiler=gcc -s compiler.version=4.9') + conaninfo = load(os.path.join(client.current_folder, "conaninfo.txt")) + self.assertNotIn("libcxx", conaninfo) + + client.run("create lasote/testing -s compiler=gcc -s compiler.version=4.9") # Now try to reuse the installed package defining libstc++11 for the new package newlib_content = ''' from conans import ConanFile, CMake class ConanFileToolsTest(ConanFile): - name = "test2" - version = "1.9" settings = "os", "compiler", "arch", "build_type" - url = "1" - license = "2" - export = ["CMakeLists.txt", "main.c"] - generators = ["cmake"] requires = "test/1.9@lasote/testing" - - def build(self): - pass ''' - new_client = TestClient(base_folder=client.base_folder) # Share storage - new_client.save({"conanfile.py": newlib_content, "CMakeLists.txt": cmakelists}) - new_client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=4.9', ignore_error=False) - # Package is found and everything is ok - self.assertIn("Generator cmake created conanbuildinfo.cmake", str(new_client.user_io.out)) - # Try again without removing the setting, if we use libstdc++11, the C package won't be found - self.files["conanfile.py"] = self.files["conanfile.py"].replace("def config", "def config222") - client.save(self.files) - client.run("export lasote/testing") - client.run("install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9") + client.save({"conanfile.py": newlib_content}) + client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=4.9') + # Package is found and everything is ok + self.assertIn("test/1.9@lasote/testing: Already installed!", client.out) + self.assertIn("PROJECT: Generated conaninfo.txt", client.out) conaninfo = load(os.path.join(client.current_folder, "conaninfo.txt")) - self.assertIn("libcxx", conaninfo[:conaninfo.find("[full_settings]")]) - client.run('install test/1.9@lasote/testing -s compiler=gcc --build -s compiler.libcxx=libstdc++ -s compiler.version=4.9', ignore_error=False) - new_client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=4.9', ignore_error=True) - self.assertIn("Can't find a 'test/1.9@lasote/testing' package for the specified options and settings", str(new_client.user_io.out)) + self.assertIn("libcxx", conaninfo) + client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9') + # Package is found and everything is ok + self.assertIn("test/1.9@lasote/testing: Already installed!", client.out) + self.assertIn("PROJECT: Generated conaninfo.txt", client.out) diff --git a/conans/test/model/build_info_test.py b/conans/test/model/build_info_test.py index 0d64ee0c32a..83694726dfe 100644 --- a/conans/test/model/build_info_test.py +++ b/conans/test/model/build_info_test.py @@ -6,6 +6,7 @@ from conans.model.env_info import DepsEnvInfo from conans.test.utils.test_files import temp_folder import platform +from conans.util.files import mkdir class BuildInfoTest(unittest.TestCase): @@ -21,13 +22,15 @@ def parse_test(self): otherlib_path [includedirs_My.Component.Lib] my_component_lib +[includedirs_My-Component-Tool] +my-component-tool """ deps_info, _ = TXTGenerator.loads(text) self.assertEqual(deps_info.includedirs, ['C:/Whenever']) self.assertEqual(deps_info["Boost"].includedirs, ['F:/ChildrenPath']) self.assertEqual(deps_info["My_Lib"].includedirs, ['mylib_path']) self.assertEqual(deps_info["My_Other_Lib"].includedirs, ['otherlib_path']) - self.assertEqual(deps_info["My.Component.Lib"].includedirs, ['my_component_lib']) + self.assertEqual(deps_info["My-Component-Tool"].includedirs, ['my-component-tool']) def help_test(self): deps_env_info = DepsEnvInfo() @@ -102,13 +105,22 @@ def configs_test(self): def cpp_info_test(self): folder = temp_folder() + mkdir(os.path.join(folder, "include")) + mkdir(os.path.join(folder, "lib")) + mkdir(os.path.join(folder, "local_bindir")) + abs_folder = temp_folder() + abs_include = os.path.join(abs_folder, "usr/include") + abs_lib = os.path.join(abs_folder, "usr/lib") + abs_bin = os.path.join(abs_folder, "usr/bin") + mkdir(abs_include) + mkdir(abs_lib) + mkdir(abs_bin) info = CppInfo(folder) - info.includedirs.append("/usr/include") - info.libdirs.append("/usr/lib") - bin_abs_dir = "C:/usr/bin" if platform.system() == "Windows" else "/tmp" - info.bindirs.append(bin_abs_dir) + info.includedirs.append(abs_include) + info.libdirs.append(abs_lib) + info.bindirs.append(abs_bin) info.bindirs.append("local_bindir") - self.assertEqual(info.include_paths, [os.path.join(folder, "include"), "/usr/include"]) - self.assertEqual(info.lib_paths, [os.path.join(folder, "lib"), "/usr/lib"]) - self.assertEqual(info.bin_paths, [os.path.join(folder, "bin"), bin_abs_dir, + self.assertEqual(info.include_paths, [os.path.join(folder, "include"), abs_include]) + self.assertEqual(info.lib_paths, [os.path.join(folder, "lib"), abs_lib]) + self.assertEqual(info.bin_paths, [abs_bin, os.path.join(folder, "local_bindir")]) diff --git a/conans/test/model/options_test.py b/conans/test/model/options_test.py index 30105fcc150..652fba9553b 100644 --- a/conans/test/model/options_test.py +++ b/conans/test/model/options_test.py @@ -150,7 +150,7 @@ def test_dumps(self): "Poco:deps_bundled=True"])) def test_sha_constant(self): - self.assertEqual(self.sut.sha({"Boost", "Poco"}), + self.assertEqual(self.sut.sha, "2442d43f1d558621069a15ff5968535f818939b5") self.sut.new_option = False self.sut["Boost"].new_option = "off" @@ -165,5 +165,5 @@ def test_sha_constant(self): "Boost:thread.multi=off", "Poco:deps_bundled=True", "Poco:new_option=0"])) - self.assertEqual(self.sut.sha({"Boost", "Poco"}), + self.assertEqual(self.sut.sha, "2442d43f1d558621069a15ff5968535f818939b5") diff --git a/conans/test/model/settings_test.py b/conans/test/model/settings_test.py index 399c91a93b4..4a962f649bb 100644 --- a/conans/test/model/settings_test.py +++ b/conans/test/model/settings_test.py @@ -18,6 +18,11 @@ def setUp(self): "os": ["Windows", "Linux"]} self.sut = Settings(data) + def test_in_contains(self): + self.sut.compiler = "Visual Studio" + self.assertTrue("Visual" in self.sut.compiler) + self.assertFalse("Visual" not in self.sut.compiler) + def test_os_split(self): settings = Settings.loads("""os: Windows: diff --git a/conans/test/model/version_ranges_test.py b/conans/test/model/version_ranges_test.py index 6b174d83b11..e710338faad 100644 --- a/conans/test/model/version_ranges_test.py +++ b/conans/test/model/version_ranges_test.py @@ -17,6 +17,16 @@ class BasicMaxVersionTest(unittest.TestCase): + def prereleases_versions_test(self): + output = TestBufferConanOutput() + result = satisfying(["1.1.1", "1.1.11", "1.1.21", "1.1.111"], "", output) + self.assertEqual(result, "1.1.111") + # prereleases are ordered + result = satisfying(["1.1.1-a.1", "1.1.1-a.11", "1.1.1-a.111", "1.1.1-a.21"], "~1.1.1-a", output) + self.assertEqual(result, "1.1.1-a.111") + result = satisfying(["1.1.1", "1.1.1-11", "1.1.1-111", "1.1.1-21"], "", output) + self.assertEqual(result, "1.1.1") + def basic_test(self): output = TestBufferConanOutput() result = satisfying(["1.1", "1.2", "1.3", "2.1"], "", output) @@ -42,7 +52,7 @@ def basic_test(self): result = satisfying(["1.6.1"], ">1.5.0,<1.6.8", output) self.assertEqual(result, "1.6.1") result = satisfying(["1.1.1", "1.1.2", "1.2", "1.2.1", "1.3", "2.1"], "<=1.2", output) - self.assertEqual(result, "1.2") + self.assertEqual(result, "1.2.1") result = satisfying(["1.1.1", "1.1.2", "1.2", "1.2.1", "1.3", "2.1"], "<1.3", output) self.assertEqual(result, "1.2.1") result = satisfying(["1.a.1", "master", "X.2", "1.2.1", "1.3", "2.1"], "1.3", output) @@ -56,9 +66,9 @@ def basic_test(self): result = satisfying(["1.3.0", "1.3.1"], "<1.3", output) self.assertEqual(result, None) result = satisfying(["1.3", "1.3.1"], "<=1.3", output) - self.assertEqual(result, "1.3") + self.assertEqual(result, "1.3.1") result = satisfying(["1.3.0", "1.3.1"], "<=1.3", output) - self.assertEqual(result, "1.3.0") + self.assertEqual(result, "1.3.1") # >2 means >=3.0.0-0 result = satisfying(["2.1"], ">2", output) self.assertEqual(result, None) @@ -213,9 +223,9 @@ def test_remote_basic(self): ('("Say/1.1@memsharded/testing", "override")', "1.1", True, False), ('("Say/0.2@memsharded/testing", "override")', "0.2", True, True), # ranges - ('"Say/[<=1.2]@memsharded/testing"', "1.1.2", False, False), + ('"Say/[<=1.2]@memsharded/testing"', "1.2.1", False, False), ('"Say/[>=0.2,<=1.0]@memsharded/testing"', "0.3", False, True), - ('("Say/[<=1.2]@memsharded/testing", "override")', "1.1.2", True, False), + ('("Say/[<=1.2]@memsharded/testing", "override")', "1.2.1", True, False), ('("Say/[>=0.2,<=1.0]@memsharded/testing", "override")', "0.3", True, True), ]) def transitive_test(self, version_range, solution, override, valid): diff --git a/conans/test/remote/auth_bearer_test.py b/conans/test/remote/auth_bearer_test.py index 43d481408f5..1f53f7b8398 100644 --- a/conans/test/remote/auth_bearer_test.py +++ b/conans/test/remote/auth_bearer_test.py @@ -51,7 +51,6 @@ class AuthorizeBearerTest(unittest.TestCase): def basic_test(self): auth = AuthorizationHeaderSpy() server = TestServer(plugins=[auth]) - server.app servers = {"default": server} client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) client.save({"conanfile.py": conanfile}) @@ -77,13 +76,13 @@ def no_signature_test(self): auth = AuthorizationHeaderSpy() retur = ReturnHandlerPlugin() server = TestServer(plugins=[auth, retur]) - server.app servers = {"default": server} client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) client.save({"conanfile.py": conanfile}) client.run("export lasote/stable") - errors = client.run("upload Hello/0.1@lasote/stable") - self.assertFalse(errors) + # Upload will fail, as conan_server is expecting a signed URL + errors = client.run("upload Hello/0.1@lasote/stable", ignore_error=True) + self.assertTrue(errors) expected_calls = [('get_conan_digest_url', None), ('check_credentials', None), diff --git a/conans/test/remote/rest_api_test.py b/conans/test/remote/rest_api_test.py index 44550528ed1..f2c15393d37 100644 --- a/conans/test/remote/rest_api_test.py +++ b/conans/test/remote/rest_api_test.py @@ -3,8 +3,6 @@ from conans.model.ref import ConanFileReference, PackageReference from conans.test.utils.test_files import hello_source_files from conans.paths import CONANFILE, CONAN_MANIFEST, CONANINFO -import sys -from conans.client.output import ConanOutput, Color from conans.model.info import ConanInfo from conans.test.server.utils.server_launcher import TestServerLauncher import requests @@ -16,6 +14,7 @@ from conans.util.files import md5, save from conans.model.manifest import FileTreeManifest from nose.plugins.attrib import attr +from conans.test.utils.tools import TestBufferConanOutput @attr('slow') @@ -34,8 +33,8 @@ def setUpClass(cls): plugins=[plugin]) cls.server.start() - cls.api = RestApiClient(ConanOutput(sys.stdout, Color), requester=requests) - cls.api.remote_url = "http://localhost:%s" % str(cls.server.port) + cls.api = RestApiClient(TestBufferConanOutput(), requester=requests) + cls.api.remote_url = "http://127.0.0.1:%s" % str(cls.server.port) # Authenticate user token = cls.api.authenticate("private_user", "private_pass") diff --git a/conans/test/keep_old_export_sources_layout_test.py b/conans/test/remove_old_export_sources_layout_test.py similarity index 84% rename from conans/test/keep_old_export_sources_layout_test.py rename to conans/test/remove_old_export_sources_layout_test.py index bb48f40038b..e2e082f8cc3 100644 --- a/conans/test/keep_old_export_sources_layout_test.py +++ b/conans/test/remove_old_export_sources_layout_test.py @@ -1,16 +1,17 @@ import unittest import os +from conans.paths import EXPORT_SOURCES_DIR_OLD from conans.util.files import tar_extract from conans.test.utils.tools import TestServer, TestClient from conans.model.ref import ConanFileReference from conans.test.utils.test_files import temp_folder -class KeepOldExportSourcesLayoutTest(unittest.TestCase): +class DoNotKeepOldExportSourcesLayoutTest(unittest.TestCase): def test_basic(self): - """ check that we generate tgz with .c_src and we handle them properly + """ check that we do not generate anymore tgz with .c_src. also, they are not present any more in the cache layout, even if they come from a .c_src tgz server file """ @@ -35,11 +36,10 @@ class MyPkg(ConanFile): folder = temp_folder() with open(sources_tgz, 'rb') as file_handler: tar_extract(file_handler, folder) - self.assertEqual(os.listdir(folder), [".c_src"]) + self.assertEqual(os.listdir(folder), ["myfile.txt"]) # Now install again client.run("install Pkg/0.1@lasote/testing --build=missing") export = client.client_cache.export(conan_reference) - self.assertNotIn(".c_src", os.listdir(export)) + self.assertNotIn(EXPORT_SOURCES_DIR_OLD, os.listdir(export)) export_sources = client.client_cache.export_sources(conan_reference) self.assertEqual(os.listdir(export_sources), ["myfile.txt"]) - \ No newline at end of file diff --git a/conans/test/server/service/authorizer_test.py b/conans/test/server/service/authorizer_test.py index 994333b204a..7b0652667e6 100644 --- a/conans/test/server/service/authorizer_test.py +++ b/conans/test/server/service/authorizer_test.py @@ -1,6 +1,6 @@ import unittest from conans.server.service.authorize import BasicAuthorizer -from conans.errors import ForbiddenException, InternalErrorException +from conans.errors import ForbiddenException, InternalErrorException, AuthenticationException from conans.model.ref import ConanFileReference, PackageReference @@ -135,6 +135,47 @@ def permissions_test(self): # Pepe can't write other package self.assertRaises(ForbiddenException, authorizer.check_write_package, "pepe", self.package_reference2) + + def authenticated_user_wildcard_permissions_test(self): + """Check that authenciated user wildcard permissions logic is ok""" + # Only authenticated users can read openssl + read_perms = [(str(self.openssl_ref), "?"), ("*/*@*/*", "*")] + # Authenticated users can write any + write_perms = [("*/*@*/*", "?")] + + authorizer = BasicAuthorizer(read_perms, write_perms) + + # READ PERMISSIONS + + # Authenticated user can read conan + authorizer.check_read_conan("pepe", self.openssl_ref) + + # Authenticated user can read package + authorizer.check_read_package("pepe", self.package_reference) + + # Anonymous user can not read conan, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_read_conan, None, self.openssl_ref) + + # Anonymous user can not read package, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_read_package, None, self.package_reference) + + # WRITE PERMISSIONS + + # Authenticated user can write conan + authorizer.check_write_conan("pepe", self.openssl_ref) + + # Authenticated user can write package + authorizer.check_write_package("pepe", self.package_reference) + + # Anonymous user can not write conan, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_write_conan, None, self.openssl_ref) + + # Anonymous user can not write package, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_write_package, None, self.package_reference) def users_test(self): """Check that lists of user names are parsed correctly""" diff --git a/conans/test/server/utils/server_launcher.py b/conans/test/server/utils/server_launcher.py index 0f29456b429..7cbe9ee2dad 100644 --- a/conans/test/server/utils/server_launcher.py +++ b/conans/test/server/utils/server_launcher.py @@ -117,7 +117,11 @@ def stop(self): def clean(self): if os.path.exists(self.storage_folder): - shutil.rmtree(self.storage_folder) + try: + shutil.rmtree(self.storage_folder) + except: + print("Can't clean the test server data, probably a server process is still opened") + if __name__ == "__main__": server = TestServerLauncher() diff --git a/conans/test/util/client_conf_test.py b/conans/test/util/client_conf_test.py index 74e57df1a49..78588189ebb 100644 --- a/conans/test/util/client_conf_test.py +++ b/conans/test/util/client_conf_test.py @@ -7,6 +7,7 @@ from conans.test.utils.test_files import temp_folder from conans.util.files import save + default_client_conf = '''[storage] path: ~/.conan/data @@ -37,3 +38,15 @@ def test_quotes(self): save(os.path.join(tmp_dir, DEFAULT_PROFILE_NAME), default_profile) config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) self.assertEqual(config.env_vars["CONAN_TRACE_FILE"], "Path/with/quotes") + + def test_proxies(self): + tmp_dir = temp_folder() + save(os.path.join(tmp_dir, CONAN_CONF), "") + config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) + self.assertEqual(None, config.proxies) + save(os.path.join(tmp_dir, CONAN_CONF), "[proxies]") + config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) + self.assertNotIn("no_proxy", config.proxies) + save(os.path.join(tmp_dir, CONAN_CONF), "[proxies]\nno_proxy=localhost") + config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) + self.assertEqual(config.proxies["no_proxy"], "localhost") diff --git a/conans/test/util/output_test.py b/conans/test/util/output_test.py index ce34ccfc6f0..c7bc27f5ca4 100644 --- a/conans/test/util/output_test.py +++ b/conans/test/util/output_test.py @@ -65,12 +65,15 @@ def unzip_output_test(self): new_out = StringIO() old_out = sys.stdout try: - tools._global_output = ConanOutput(new_out) + import requests + import conans + + conans.tools.set_global_instances(ConanOutput(new_out), requests) tools.unzip(zip_path, output_dir) finally: - sys.stdout = old_out + conans.tools.set_global_instances(ConanOutput(old_out), requests) output = new_out.getvalue() - self.assertRegexpMatches(output, "Unzipping [\d]+B, this can take a while") + self.assertRegexpMatches(output, "Unzipping [\d]+B") content = load(os.path.join(output_dir, "example.txt")) self.assertEqual(content, "Hello world!") diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py index 3fcf13b3572..4af2636ca7d 100644 --- a/conans/test/util/tools_test.py +++ b/conans/test/util/tools_test.py @@ -95,7 +95,7 @@ def build(self): # Not test the real commmand get_command if it's setting the module global vars tools._global_requester = None tools._global_output = None - tmp = tempfile.mkdtemp() + tmp = temp_folder() conf = default_client_conf.replace("\n[proxies]", "\n[proxies]\nhttp = http://myproxy.com") os.mkdir(os.path.join(tmp, ".conan")) save(os.path.join(tmp, ".conan", CONAN_CONF), conf) diff --git a/conans/test/utils/cpp_test_files.py b/conans/test/utils/cpp_test_files.py index ee15cfc9f5d..8daeba08f77 100644 --- a/conans/test/utils/cpp_test_files.py +++ b/conans/test/utils/cpp_test_files.py @@ -365,6 +365,7 @@ def cpp_hello_conan_files(name="Hello", version="0.1", deps=None, language=0, st if not config: conanfile = conanfile.replace("config(", "config2(") if collect_libs: - conanfile = conanfile.replace('["hello%s"]' % name, "self.collect_libs()") + conanfile = "from conans import tools\n" + conanfile.replace('["hello%s"]' % name, + "tools.collect_libs(self)") base_files[CONANFILE] = conanfile return base_files diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py index 34141517df3..b5261a3de9d 100644 --- a/conans/test/utils/tools.py +++ b/conans/test/utils/tools.py @@ -35,8 +35,7 @@ TestServerLauncher) from conans.test.utils.runner import TestRunner from conans.test.utils.test_files import temp_folder -from conans.util.env_reader import get_env -from conans.util.files import save_files, load, save +from conans.util.files import save_files, save from conans.util.log import logger @@ -295,7 +294,7 @@ class TestClient(object): def __init__(self, base_folder=None, current_folder=None, servers=None, users=None, client_version=CLIENT_VERSION, min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION, - requester_class=None, runner=None, path_with_spaces=True): + requester_class=None, runner=None, path_with_spaces=True, default_profile=True): """ storage_folder: Local storage path current_folder: Current execution folder @@ -335,6 +334,13 @@ def __init__(self, base_folder=None, current_folder=None, logger.debug("Client storage = %s" % self.storage_folder) self.current_folder = current_folder or temp_folder(path_with_spaces) + # Enforcing VS 2015, even if VS2017 is auto detected + if default_profile: + profile = self.client_cache.default_profile + if profile.settings.get("compiler.version") == "15": + profile.settings["compiler.version"] = "14" + save(self.client_cache.default_profile_path, profile.dumps()) + @property def paths(self): return self.client_cache diff --git a/conans/tools.py b/conans/tools.py index f6ce34c77fa..45d360092f8 100644 --- a/conans/tools.py +++ b/conans/tools.py @@ -1,800 +1,28 @@ -""" ConanFile user tools, as download, etc -""" +""" ConanFile user tools, as download, etc""" from __future__ import print_function -import logging -import multiprocessing -import os -import platform -import re -import subprocess -import sys - - -from contextlib import contextmanager - import requests -from patch import fromfile, fromstring - +from conans.client.tools import * from conans.client.output import ConanOutput -from conans.client.rest.uploader_downloader import Downloader -from conans.client.runner import ConanRunner -from conans.errors import ConanException -from conans.model.version import Version # noinspection PyUnresolvedReferences -from conans.util.files import _generic_algorithm_sum, load, save, sha256sum, \ - sha1sum, md5sum, md5, touch, relative_dirs, rmdir, mkdir -from conans.util.log import logger - -# Default values -_global_requester = requests -_global_output = ConanOutput(sys.stdout) - - -def unix_path(path): - """"Used to translate windows paths to MSYS unix paths like - c/users/path/to/file""" - pattern = re.compile(r'([a-z]):\\', re.IGNORECASE) - return pattern.sub('/\\1/', path).replace('\\', '/').lower() - - -def escape_windows_cmd(command): - """ To use in a regular windows cmd.exe - 1. Adds escapes so the argument can be unpacked by CommandLineToArgvW() - 2. Adds escapes for cmd.exe so the argument survives cmd.exe's substitutions. - - Useful to escape commands to be executed in a windows bash (msys2, cygwin etc) - """ - quoted_arg = subprocess.list2cmdline([command]) - return "".join(["^%s" % arg if arg in r'()%!^"<>&|' else arg for arg in quoted_arg]) - - -def run_in_windows_bash(conanfile, bashcmd, cwd=None): - """ Will run a unix command inside the msys2 environment - It requires to have MSYS2 in the path and MinGW - """ - if platform.system() != "Windows": - raise ConanException("Command only for Windows operating system") - # This needs to be set so that msys2 bash profile will set up the environment correctly. - try: - arch = conanfile.settings.arch # Maybe arch doesn't exist - except: - arch = None - env_vars = {"MSYSTEM": "MINGW32" if arch == "x86" else "MINGW64", - "MSYS2_PATH_TYPE": "inherit"} - with environment_append(env_vars): - curdir = unix_path(cwd or os.path.abspath(os.path.curdir)) - # Needed to change to that dir inside the bash shell - to_run = 'cd "%s" && %s ' % (curdir, bashcmd) - custom_bash_path = os.getenv("CONAN_BASH_PATH", "bash") - wincmd = '%s --login -c %s' % (custom_bash_path, escape_windows_cmd(to_run)) - conanfile.output.info('run_in_windows_bash: %s' % wincmd) - conanfile.run(wincmd) - - -def args_to_string(args): - if not args: - return "" - if sys.platform == 'win32': - return subprocess.list2cmdline(args) - else: - return " ".join("'" + arg.replace("'", r"'\''") + "'" for arg in args) - - -@contextmanager -def chdir(newdir): - old_path = os.getcwd() - os.chdir(newdir) - try: - yield - finally: - os.chdir(old_path) - - -@contextmanager -def pythonpath(conanfile): - old_path = sys.path[:] - python_path = conanfile.env.get("PYTHONPATH", None) - if python_path: - if isinstance(python_path, list): - sys.path.extend(python_path) - else: - sys.path.append(python_path) - - yield - sys.path = old_path - - -@contextmanager -def environment_append(env_vars): - """ - :param env_vars: List of simple environment vars. {name: value, name2: value2} => e.j: MYVAR=1 - The values can also be lists of appendable environment vars. {name: [value, value2]} - => e.j. PATH=/path/1:/path/2 - :return: None - """ - old_env = dict(os.environ) - for name, value in env_vars.items(): - if isinstance(value, list): - env_vars[name] = os.pathsep.join(value) - if name in old_env: - env_vars[name] += os.pathsep + old_env[name] - os.environ.update(env_vars) - try: - yield - finally: - os.environ.clear() - os.environ.update(old_env) - - -def msvc_build_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, - arch=None): - """ Do both: set the environment variables and call the .sln build - """ - vcvars = vcvars_command(settings) - build = build_sln_command(settings, sln_path, targets, upgrade_project, build_type, arch) - command = "%s && %s" % (vcvars, build) - return command - - -def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, - arch=None): - """ - Use example: - build_command = build_sln_command(self.settings, "myfile.sln", targets=["SDL2_image"]) - command = "%s && %s" % (tools.vcvars_command(self.settings), build_command) - self.run(command) - """ - targets = targets or [] - command = "devenv %s /upgrade && " % sln_path if upgrade_project else "" - build_type = build_type or settings.build_type - arch = arch or settings.arch - if not build_type: - raise ConanException("Cannot build_sln_command, build_type not defined") - if not arch: - raise ConanException("Cannot build_sln_command, arch not defined") - command += "msbuild %s /p:Configuration=%s" % (sln_path, build_type) - arch = str(arch) - if arch in ["x86_64", "x86"]: - command += ' /p:Platform=' - command += '"x64"' if arch == "x86_64" else '"x86"' - elif "ARM" in arch.upper(): - command += ' /p:Platform="ARM"' - - if targets: - command += " /target:%s" % ";".join(targets) - return command - - -def vs_installation_path(version): - if not hasattr(vs_installation_path, "_cached"): - vs_installation_path._cached = dict() - - if version not in vs_installation_path._cached: - vs_path = None - program_files = os.environ.get("ProgramFiles(x86)", os.environ.get("ProgramFiles")) - if program_files: - vswhere_path = os.path.join(program_files, "Microsoft Visual Studio", "Installer", - "vswhere.exe") - if os.path.isfile(vswhere_path): - version_range = "[%d.0, %d.0)" % (int(version), int(version) + 1) - try: - output = subprocess.check_output([vswhere_path, "-version", version_range, - "-legacy", "-property", "installationPath"]) - vs_path = output.decode().strip() - _global_output.info("vswhere detected VS %s in %s" % (version, vs_path)) - except (ValueError, subprocess.CalledProcessError, UnicodeDecodeError) as e: - _global_output.error("vswhere error: %s" % str(e)) - - # Remember to cache result - vs_installation_path._cached[version] = vs_path - - return vs_installation_path._cached[version] - - -def vcvars_command(settings): - arch_setting = settings.get_safe("arch") - compiler_version = settings.get_safe("compiler.version") - if not compiler_version: - raise ConanException("compiler.version setting required for vcvars not defined") - - param = "x86" if arch_setting == "x86" else "amd64" - existing_version = os.environ.get("VisualStudioVersion") - if existing_version: - command = "echo Conan:vcvars already set" - existing_version = existing_version.split(".")[0] - if existing_version != compiler_version: - raise ConanException("Error, Visual environment already set to %s\n" - "Current settings visual version: %s" - % (existing_version, compiler_version)) - else: - env_var = "vs%s0comntools" % compiler_version - - if env_var == 'vs150comntools': - vs_path = os.getenv(env_var) - if not vs_path: # Try to locate with vswhere - vs_root = vs_installation_path("15") - if vs_root: - vs_path = os.path.join(vs_root, "Common7", "Tools") - else: - raise ConanException("VS2017 '%s' variable not defined, " - "and vswhere didn't find it" % env_var) - vcvars_path = os.path.join(vs_path, "../../VC/Auxiliary/Build/vcvarsall.bat") - command = ('set "VSCMD_START_DIR=%%CD%%" && ' - 'call "%s" %s' % (vcvars_path, param)) - else: - try: - vs_path = os.environ[env_var] - except KeyError: - raise ConanException("VS '%s' variable not defined. Please install VS" % env_var) - vcvars_path = os.path.join(vs_path, "../../VC/vcvarsall.bat") - command = ('call "%s" %s' % (vcvars_path, param)) - - return command - - -def cpu_count(): - try: - env_cpu_count = os.getenv("CONAN_CPU_COUNT", None) - return int(env_cpu_count) if env_cpu_count else multiprocessing.cpu_count() - except NotImplementedError: - _global_output.warn("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") - return 1 # Safe guess - - -def human_size(size_bytes): - """ - format a size in bytes into a 'human' file size, e.g. B, KB, MB, GB, TB, PB - Note that bytes will be reported in whole numbers but KB and above will have - greater precision. e.g. 43 B, 443 KB, 4.3 MB, 4.43 GB, etc - """ - - suffixes_table = [('B', 0), ('KB', 1), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)] - - num = float(size_bytes) - for suffix, precision in suffixes_table: - if num < 1024.0: - break - num /= 1024.0 - - if precision == 0: - formatted_size = "%d" % num - else: - formatted_size = str(round(num, ndigits=precision)) - - return "%s%s" % (formatted_size, suffix) - - -def unzip(filename, destination=".", keep_permissions=False): - """ - Unzip a zipped file - :param filename: Path to the zip file - :param destination: Destination folder - :param keep_permissions: Keep the zip permissions. WARNING: Can be dangerous if the zip was not created in a NIX - system, the bits could produce undefined permission schema. Use only this option if you are sure that the - zip was created correctly. - :return: - """ - if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or - filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or - filename.endswith(".tar")): - return untargz(filename, destination) - import zipfile - full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) - - if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): - def print_progress(extracted_size, uncompress_size): - txt_msg = "Unzipping %.0f %%" % (extracted_size * 100.0 / uncompress_size) - _global_output.rewrite_line(txt_msg) - else: - def print_progress(extracted_size, uncompress_size): - pass - - with zipfile.ZipFile(filename, "r") as z: - uncompress_size = sum((file_.file_size for file_ in z.infolist())) - _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) - extracted_size = 0 - if platform.system() == "Windows": - for file_ in z.infolist(): - extracted_size += file_.file_size - print_progress(extracted_size, uncompress_size) - try: - # Win path limit is 260 chars - if len(file_.filename) + len(full_path) >= 260: - raise ValueError("Filename too long") - z.extract(file_, full_path) - except Exception as e: - _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) - else: # duplicated for, to avoid a platform check for each zipped file - for file_ in z.infolist(): - extracted_size += file_.file_size - print_progress(extracted_size, uncompress_size) - try: - z.extract(file_, full_path) - if keep_permissions: - # Could be dangerous if the ZIP has been created in a non nix system - # https://bugs.python.org/issue15795 - perm = file_.external_attr >> 16 & 0xFFF - os.chmod(os.path.join(full_path, file_.filename), perm) - except Exception as e: - _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) - - -def untargz(filename, destination="."): - import tarfile - with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile: - tarredgzippedFile.extractall(destination) - - -def get(url): - """ high level downloader + unziper + delete temporary zip - """ - filename = os.path.basename(url) - download(url, filename) - unzip(filename) - os.unlink(filename) - - -def ftp_download(ip, filename, login='', password=''): - import ftplib - try: - ftp = ftplib.FTP(ip, login, password) - ftp.login() - filepath, filename = os.path.split(filename) - if filepath: - ftp.cwd(filepath) - with open(filename, 'wb') as f: - ftp.retrbinary('RETR ' + filename, f.write) - except Exception as e: - raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e))) - finally: - try: - ftp.quit() - except: - pass - - -def download(url, filename, verify=True, out=None, retry=2, retry_wait=5): - out = out or ConanOutput(sys.stdout, True) - if verify: - # We check the certificate using a list of known verifiers - import conans.client.rest.cacert as cacert - verify = cacert.file_path - downloader = Downloader(_global_requester, out, verify=verify) - downloader.download(url, filename, retry=retry, retry_wait=retry_wait) - out.writeln("") - - -# save(filename, content) - - -def replace_in_file(file_path, search, replace): - content = load(file_path) - content = content.replace(search, replace) - content = content.encode("utf-8") - with open(file_path, "wb") as handle: - handle.write(content) - - -def replace_prefix_in_pc_file(pc_file, new_prefix): - content = load(pc_file) - lines = [] - for line in content.splitlines(): - if line.startswith("prefix="): - lines.append('prefix=%s' % new_prefix) - else: - lines.append(line) - save(pc_file, "\n".join(lines)) - - -def check_with_algorithm_sum(algorithm_name, file_path, signature): - real_signature = _generic_algorithm_sum(file_path, algorithm_name) - if real_signature != signature: - raise ConanException("%s signature failed for '%s' file." - " Computed signature: %s" % (algorithm_name, - os.path.basename(file_path), - real_signature)) - - -def check_sha1(file_path, signature): - check_with_algorithm_sum("sha1", file_path, signature) - - -def check_md5(file_path, signature): - check_with_algorithm_sum("md5", file_path, signature) - - -def check_sha256(file_path, signature): - check_with_algorithm_sum("sha256", file_path, signature) - - -def patch(base_path=None, patch_file=None, patch_string=None, strip=0, output=None): - """Applies a diff from file (patch_file) or string (patch_string) - in base_path directory or current dir if None""" - - class PatchLogHandler(logging.Handler): - def __init__(self): - logging.Handler.__init__(self, logging.DEBUG) - self.output = output or ConanOutput(sys.stdout, True) - self.patchname = patch_file if patch_file else "patch" - - def emit(self, record): - logstr = self.format(record) - if record.levelno == logging.WARN: - self.output.warn("%s: %s" % (self.patchname, logstr)) - else: - self.output.info("%s: %s" % (self.patchname, logstr)) - - patchlog = logging.getLogger("patch") - if patchlog: - patchlog.handlers = [] - patchlog.addHandler(PatchLogHandler()) - - if not patch_file and not patch_string: - return - if patch_file: - patchset = fromfile(patch_file) - else: - patchset = fromstring(patch_string.encode()) - - if not patchset: - raise ConanException("Failed to parse patch: %s" % (patch_file if patch_file else "string")) - - if not patchset.apply(root=base_path, strip=strip): - raise ConanException("Failed to apply patch: %s" % patch_file) - - -def cross_building(settings, self_os=None, self_arch=None): - self_os = self_os or platform.system() - self_arch = self_arch or detected_architecture() - os_setting = settings.get_safe("os") - arch_setting = settings.get_safe("arch") - platform_os = {"Darwin": "Macos"}.get(self_os, self_os) - if self_os == os_setting and self_arch == "x86_64" and arch_setting == "x86": - return False # not really considered cross - - if os_setting and platform_os != os_setting: - return True - if arch_setting and self_arch != arch_setting: - return True - - return False - - -def detected_architecture(): - # FIXME: Very weak check but not very common to run conan in other architectures - if "64" in platform.machine(): - return "x86_64" - elif "86" in platform.machine(): - return "x86" - return None - - -# DETECT OS, VERSION AND DISTRIBUTIONS - -class OSInfo(object): - """ Usage: - (os_info.is_linux) # True/False - (os_info.is_windows) # True/False - (os_info.is_macos) # True/False - (os_info.is_freebsd) # True/False - (os_info.is_solaris) # True/False - - (os_info.linux_distro) # debian, ubuntu, fedora, centos... - - (os_info.os_version) # 5.1 - (os_info.os_version_name) # Windows 7, El Capitan - - if os_info.os_version > "10.1": - pass - if os_info.os_version == "10.1.0": - pass - """ - - def __init__(self): - self.os_version = None - self.os_version_name = None - self.is_linux = platform.system() == "Linux" - self.linux_distro = None - self.is_windows = platform.system() == "Windows" - self.is_macos = platform.system() == "Darwin" - self.is_freebsd = platform.system() == "FreeBSD" - self.is_solaris = platform.system() == "SunOS" - - if self.is_linux: - import distro - self.linux_distro = distro.id() - self.os_version = Version(distro.version()) - version_name = distro.codename() - self.os_version_name = version_name if version_name != "n/a" else "" - if not self.os_version_name and self.linux_distro == "debian": - self.os_version_name = self.get_debian_version_name(self.os_version) - elif self.is_windows: - self.os_version = self.get_win_os_version() - self.os_version_name = self.get_win_version_name(self.os_version) - elif self.is_macos: - self.os_version = Version(platform.mac_ver()[0]) - self.os_version_name = self.get_osx_version_name(self.os_version) - elif self.is_freebsd: - self.os_version = self.get_freebsd_version() - self.os_version_name = "FreeBSD %s" % self.os_version - elif self.is_solaris: - self.os_version = Version(platform.release()) - self.os_version_name = self.get_solaris_version_name(self.os_version) - - @property - def with_apt(self): - return self.is_linux and self.linux_distro in \ - ("debian", "ubuntu", "knoppix", "linuxmint", "raspbian") - - @property - def with_yum(self): - return self.is_linux and self.linux_distro in \ - ("centos", "redhat", "fedora", "pidora", "scientific", - "xenserver", "amazon", "oracle", "rhel") - - @staticmethod - def get_win_os_version(): - """ - Get's the OS major and minor versions. Returns a tuple of - (OS_MAJOR, OS_MINOR). - """ - import ctypes - - class _OSVERSIONINFOEXW(ctypes.Structure): - _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), - ('dwMajorVersion', ctypes.c_ulong), - ('dwMinorVersion', ctypes.c_ulong), - ('dwBuildNumber', ctypes.c_ulong), - ('dwPlatformId', ctypes.c_ulong), - ('szCSDVersion', ctypes.c_wchar * 128), - ('wServicePackMajor', ctypes.c_ushort), - ('wServicePackMinor', ctypes.c_ushort), - ('wSuiteMask', ctypes.c_ushort), - ('wProductType', ctypes.c_byte), - ('wReserved', ctypes.c_byte)] - - os_version = _OSVERSIONINFOEXW() - os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) - retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) - if retcode != 0: - return None - - return Version("%d.%d" % (os_version.dwMajorVersion, os_version.dwMinorVersion)) - - @staticmethod - def get_debian_version_name(version): - if not version: - return None - elif version.major() == "8.Y.Z": - return "jessie" - elif version.major() == "7.Y.Z": - return "wheezy" - elif version.major() == "6.Y.Z": - return "squeeze" - elif version.major() == "5.Y.Z": - return "lenny" - elif version.major() == "4.Y.Z": - return "etch" - elif version.minor() == "3.1.Z": - return "sarge" - elif version.minor() == "3.0.Z": - return "woody" - - @staticmethod - def get_win_version_name(version): - if not version: - return None - elif version.major() == "5.Y.Z": - return "Windows XP" - elif version.minor() == "6.0.Z": - return "Windows Vista" - elif version.minor() == "6.1.Z": - return "Windows 7" - elif version.minor() == "6.2.Z": - return "Windows 8" - elif version.minor() == "6.3.Z": - return "Windows 8.1" - elif version.minor() == "10.0.Z": - return "Windows 10" - - @staticmethod - def get_osx_version_name(version): - if not version: - return None - elif version.minor() == "10.12.Z": - return "Sierra" - elif version.minor() == "10.11.Z": - return "El Capitan" - elif version.minor() == "10.10.Z": - return "Yosemite" - elif version.minor() == "10.9.Z": - return "Mavericks" - elif version.minor() == "10.8.Z": - return "Mountain Lion" - elif version.minor() == "10.7.Z": - return "Lion" - elif version.minor() == "10.6.Z": - return "Snow Leopard" - elif version.minor() == "10.5.Z": - return "Leopard" - elif version.minor() == "10.4.Z": - return "Tiger" - elif version.minor() == "10.3.Z": - return "Panther" - elif version.minor() == "10.2.Z": - return "Jaguar" - elif version.minor() == "10.1.Z": - return "Puma" - elif version.minor() == "10.0.Z": - return "Cheetha" - - @staticmethod - def get_freebsd_version(): - return platform.release().split("-")[0] - - @staticmethod - def get_solaris_version_name(version): - if not version: - return None - elif version.minor() == "5.10": - return "Solaris 10" - elif version.minor() == "5.11": - return "Solaris 11" - - -try: - os_info = OSInfo() -except Exception as exc: - logger.error(exc) - _global_output.error("Error detecting os_info") - - -class SystemPackageTool(object): - - def __init__(self, runner=None, os_info=None, tool=None): - env_sudo = os.environ.get("CONAN_SYSREQUIRES_SUDO", None) - self._sudo = (env_sudo != "False" and env_sudo != "0") - os_info = os_info or OSInfo() - self._is_up_to_date = False - self._tool = tool or self._create_tool(os_info) - self._tool._sudo_str = "sudo " if self._sudo else "" - self._tool._runner = runner or ConanRunner() - - def _create_tool(self, os_info): - if os_info.with_apt: - return AptTool() - elif os_info.with_yum: - return YumTool() - elif os_info.is_macos: - return BrewTool() - elif os_info.is_freebsd: - return PkgTool() - elif os_info.is_solaris: - return PkgUtilTool() - else: - return NullTool() - - def update(self): - """ - Get the system package tool update command - """ - self._is_up_to_date = True - self._tool.update() - - def install(self, packages, update=True, force=False): - ''' - Get the system package tool install command. - ''' - packages = [packages] if isinstance(packages, str) else list(packages) - if not force and self._installed(packages): - return - if update and not self._is_up_to_date: - self.update() - self._install_any(packages) - - def _installed(self, packages): - for pkg in packages: - if self._tool.installed(pkg): - _global_output.info("Package already installed: %s" % pkg) - return True - return False - - def _install_any(self, packages): - if len(packages) == 1: - return self._tool.install(packages[0]) - for pkg in packages: - try: - return self._tool.install(pkg) - except ConanException: - pass - raise ConanException("Could not install any of %s" % packages) - - -class NullTool(object): - def update(self): - pass - - def install(self, package_name): - _global_output.warn("Only available for linux with apt-get or yum or OSx with brew or FreeBSD with pkg or Solaris with pkgutil") - - def installed(self, package_name): - return False - - -class AptTool(object): - def update(self): - _run(self._runner, "%sapt-get update" % self._sudo_str) - - def install(self, package_name): - _run(self._runner, "%sapt-get install -y %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner("dpkg -s %s" % package_name, None) - return exit_code == 0 - - -class YumTool(object): - def update(self): - _run(self._runner, "%syum check-update" % self._sudo_str, accepted_returns=[0, 100]) - - def install(self, package_name): - _run(self._runner, "%syum install -y %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner("rpm -q %s" % package_name, None) - return exit_code == 0 - - -class BrewTool(object): - def update(self): - _run(self._runner, "brew update") - - def install(self, package_name): - _run(self._runner, "brew install %s" % package_name) - - def installed(self, package_name): - exit_code = self._runner('test -n "$(brew ls --versions %s)"' % package_name, None) - return exit_code == 0 - - -class PkgTool(object): - def update(self): - _run(self._runner, "%spkg update" % self._sudo_str) - - def install(self, package_name): - _run(self._runner, "%spkg install -y %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner("pkg info %s" % package_name, None) - return exit_code == 0 - - -class PkgUtilTool(object): - def update(self): - _run(self._runner, "%spkgutil --catalog" % self._sudo_str) - - def install(self, package_name): - _run(self._runner, "%spkgutil --install --yes %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner('test -n "`pkgutil --list %s`"' % package_name, None) - return exit_code == 0 +from conans.util.files import (_generic_algorithm_sum, load, save, sha256sum, + sha1sum, md5sum, md5, touch, relative_dirs, rmdir, mkdir) -class ChocolateyTool(object): - def update(self): - _run(self._runner, "choco outdated") - def install(self, package_name): - _run(self._runner, "choco install --yes %s" % package_name) +# Global instances +def set_global_instances(the_output, the_requester): + # Assign global variables to needed modules + from conans.client.tools import files as _files + from conans.client.tools import net as _net + from conans.client.tools import oss as _oss + from conans.client.tools import system_pm as _system_pm + from conans.client.tools import win as _win - def installed(self, package_name): - exit_code = self._runner('choco search --local-only --exact %s | findstr /c:"1 packages installed."' % package_name, None) - return exit_code == 0 + _files._global_output = the_output + _oss._global_output = the_output + _system_pm._global_output = the_output + _win._global_output = the_output + _net._global_requester = the_requester -def _run(runner, command, accepted_returns=None): - accepted_returns = accepted_returns or [0, ] - _global_output.info("Running: %s" % command) - if runner(command, True) not in accepted_returns: - raise ConanException("Command '%s' failed" % command) +set_global_instances(ConanOutput(sys.stdout), requests) diff --git a/conans/util/files.py b/conans/util/files.py index ad9765e3393..3dbc734e4a4 100644 --- a/conans/util/files.py +++ b/conans/util/files.py @@ -10,6 +10,23 @@ from conans.util.log import logger import tarfile +_DIRTY_FOLDER = ".dirty" + + +def set_dirty(folder): + dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER + save(dirty_file, "") + + +def clean_dirty(folder): + dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER + os.remove(dirty_file) + + +def is_dirty(folder): + dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER + return os.path.exists(dirty_file) + def decode_text(text): decoders = ["utf-8", "Windows-1252"] diff --git a/conans/util/windows.py b/conans/util/windows.py new file mode 100644 index 00000000000..72112433ccb --- /dev/null +++ b/conans/util/windows.py @@ -0,0 +1,83 @@ +import os +from conans.util.files import load, mkdir, save, rmdir +import tempfile + + +CONAN_LINK = ".conan_link" + + +def conan_expand_user(path): + """ wrapper to the original expanduser function, to workaround python returning + verbatim %USERPROFILE% when some other app (git for windows) sets HOME envvar + """ + # In win these variables should exist and point to user directory, which + # must exist. Using context to avoid permanent modification of os.environ + old_env = dict(os.environ) + try: + home = os.environ.get("HOME") + # Problematic cases of wrong HOME variable + # - HOME = %USERPROFILE% verbatim, as messed by some other tools + # - MSYS console, that defines a different user home in /c/mingw/msys/users/xxx + # In these cases, it is safe to remove it and rely on USERPROFILE directly + if home and (not os.path.exists(home) or + (os.getenv("MSYSTEM") and os.getenv("USERPROFILE"))): + del os.environ["HOME"] + result = os.path.expanduser(path) + finally: + os.environ.clear() + os.environ.update(old_env) + return result + + +def path_shortener(path, short_paths): + """ short_paths is 4-state: + False: Never shorten the path + True: Always shorten the path, create link if not existing + None: Use shorten path only if already exists, not create + """ + if short_paths is False or os.getenv("CONAN_USER_HOME_SHORT") == "None": + return path + link = os.path.join(path, CONAN_LINK) + if os.path.exists(link): + return load(link) + elif short_paths is None: + return path + + short_home = os.getenv("CONAN_USER_HOME_SHORT") + if not short_home: + drive = os.path.splitdrive(path)[0] + short_home = drive + "/.conan" + mkdir(short_home) + redirect = tempfile.mkdtemp(dir=short_home, prefix="") + # This "1" is the way to have a non-existing directory, so commands like + # shutil.copytree() to it, works. It can be removed without compromising the + # temp folder generator and conan-links consistency + redirect = os.path.join(redirect, "1") + save(link, redirect) + return redirect + + +def ignore_long_path_files(src_folder, build_folder, output): + def _filter(src, files): + filtered_files = [] + for the_file in files: + source_path = os.path.join(src, the_file) + # Without storage path, just relative + rel_path = os.path.relpath(source_path, src_folder) + dest_path = os.path.normpath(os.path.join(build_folder, rel_path)) + # it is NOT that "/" is counted as "\\" so it counts double + # seems a bug in python, overflows paths near the limit of 260, + if len(dest_path) >= 249: + filtered_files.append(the_file) + output.warn("Filename too long, file excluded: %s" % dest_path) + return filtered_files + return _filter + + +def rm_conandir(path): + """removal of a directory that might contain a link to a short path""" + link = os.path.join(path, CONAN_LINK) + if os.path.exists(link): + short_path = load(link) + rmdir(os.path.dirname(short_path)) + rmdir(path) diff --git a/contributors.txt b/contributors.txt index 8a088adb60c..857e589da6b 100644 --- a/contributors.txt +++ b/contributors.txt @@ -16,7 +16,9 @@ Many thanks to all of them! - Koch, Marco (marco-koch@t-online.de, @MarcoKoch) - Kourkoulis, Dimitri (@dimi309) - Lee, Jeongseok (jslee02@gmail.com, @jslee02) +- Lord, Matthew( @luckielordie ) - Márki, Róbert (gsmiko@gmail.com, @robertmrk) - Ray, Chris (chris@xaltotun.com) - Ries, Uilian (uilianries@gmail.com, @uilianries) - Sechet, Olivier (osechet@gmail.com) +- Sturm, Fabian (f@rtfs.org, @sturmf)