From 35d8d91b417520ac9e5e76111f688c6acfa13d59 Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 3 Jul 2024 13:06:01 +0200 Subject: [PATCH] Added XXXX_FOR_BUILD variables and android extra flags mechanism --- conan/tools/gnu/gnutoolchain.py | 113 ++++++++++++++---- .../toolchains/gnu/autotools/test_android.py | 3 +- .../gnu/test_gnutoolchain_android.py | 50 ++++++++ .../toolchains/gnu/test_gnutoolchain.py | 19 +-- 4 files changed, 154 insertions(+), 31 deletions(-) create mode 100644 test/functional/toolchains/gnu/test_gnutoolchain_android.py diff --git a/conan/tools/gnu/gnutoolchain.py b/conan/tools/gnu/gnutoolchain.py index 95c32129e9d..62a1176ff06 100644 --- a/conan/tools/gnu/gnutoolchain.py +++ b/conan/tools/gnu/gnutoolchain.py @@ -1,3 +1,5 @@ +import os + from conan.internal import check_duplicated_generator from conan.internal.internal_tools import raise_on_universal_arch from conan.tools.apple.apple import is_apple_os, resolve_apple_flags @@ -9,6 +11,7 @@ from conan.tools.env import Environment from conan.tools.gnu.get_gnu_triplet import _get_gnu_triplet from conan.tools.microsoft import VCVars, msvc_runtime_flag, unix_path, check_min_vs, is_msvc +from conans.errors import ConanException from conans.model.pkg_type import PackageType @@ -18,6 +21,8 @@ class GnuToolchain: Note: it's based on legacy AutotoolsToolchain but with a more modern and usable UX """ + script_name = "conangnutoolchain" + def __init__(self, conanfile, namespace=None, prefix="/"): """ :param conanfile: The current recipe object. Always use ``self``. @@ -63,8 +68,8 @@ def __init__(self, conanfile, namespace=None, prefix="/"): "host": {"triplet": self._conanfile.conf.get("tools.gnu:host_triplet")}, "build": {"triplet": self._conanfile.conf.get("tools.gnu:build_triplet")} } - is_cross_building = cross_building(self._conanfile) - if is_cross_building: + self._is_cross_building = cross_building(self._conanfile) + if self._is_cross_building: compiler = self._conanfile.settings.get_safe("compiler") # Host triplet if not self.triplets_info["host"]["triplet"]: @@ -88,7 +93,7 @@ def __init__(self, conanfile, namespace=None, prefix="/"): self.configure_args.update(self._get_default_configure_install_flags()) self.configure_args.update(self._get_default_triplets()) # Apple stuff - is_cross_building_osx = (is_cross_building + is_cross_building_osx = (self._is_cross_building and conanfile.settings_build.get_safe('os') == "Macos" and is_apple_os(conanfile)) min_flag, arch_flag, isysroot_flag = ( @@ -99,7 +104,7 @@ def __init__(self, conanfile, namespace=None, prefix="/"): # -isysroot makes all includes for your library relative to the build directory self.apple_isysroot_flag = isysroot_flag self.apple_min_version_flag = min_flag - # MSVC common stuff + # Default initial environment flags self._initialize_default_extra_env() def yes_no(self, option_name, default=None, negated=False): @@ -116,33 +121,99 @@ def yes_no(self, option_name, default=None, negated=False): option_value = not option_value if negated else option_value return "yes" if option_value else "no" - def _initialize_default_extra_env(self): - """Initialize the default environment variables.""" - extra_env_vars = dict() - # Normally, these are the most common default flags used by MSVC in Windows - if is_msvc(self._conanfile): - extra_env_vars = {"CC": "cl -nologo", - "CXX": "cl -nologo", - "LD": "link -nologo", - "AR": "lib", - "NM": "dumpbin -symbols", - "OBJDUMP": ":", - "RANLIB": ":", - "STRIP": ":"} + def _resolve_android_cross_compilation(self): + # Issue related: https://github.com/conan-io/conan/issues/13443 + ret = {} + if not self._is_cross_building or not self._conanfile.settings.get_safe("os") == "Android": + return ret + + ndk_path = self._conanfile.conf.get("tools.android:ndk_path", check_type=str) + if not ndk_path: + raise ConanException("You must provide a NDK path. Use 'tools.android:ndk_path' " + "configuration field.") + + if ndk_path: + arch = self._conanfile.settings.get_safe("arch") + os_build = self._conanfile.settings_build.get_safe("os") + ndk_os_folder = { + 'Macos': 'darwin', + 'iOS': 'darwin', + 'watchOS': 'darwin', + 'tvOS': 'darwin', + 'visionOS': 'darwin', + 'FreeBSD': 'linux', + 'Linux': 'linux', + 'Windows': 'windows', + 'WindowsCE': 'windows', + 'WindowsStore': 'windows' + }.get(os_build, "linux") + ndk_bin = os.path.join(ndk_path, "toolchains", "llvm", "prebuilt", + f"{ndk_os_folder}-x86_64", "bin") + android_api_level = self._conanfile.settings.get_safe("os.api_level") + android_target = {'armv7': 'armv7a-linux-androideabi', + 'armv8': 'aarch64-linux-android', + 'x86': 'i686-linux-android', + 'x86_64': 'x86_64-linux-android'}.get(arch) + os_build = self._conanfile.settings_build.get_safe('os') + ext = ".cmd" if os_build == "Windows" else "" + ret = { + "CC": os.path.join(ndk_bin, f"{android_target}{android_api_level}-clang{ext}"), + "CXX": os.path.join(ndk_bin, f"{android_target}{android_api_level}-clang++{ext}"), + "LD": os.path.join(ndk_bin, "ld"), + "STRIP": os.path.join(ndk_bin, "llvm-strip"), + "RANLIB": os.path.join(ndk_bin, "llvm-ranlib"), + "AS": os.path.join(ndk_bin, f"{android_target}{android_api_level}-clang{ext}"), + "AR": os.path.join(ndk_bin, "llvm-ar"), + } + # Overriding host triplet + self.triplets_info["host"]["triplet"] = android_target + return ret + + def _resolve_compilers_mapping_variables(self): + ret = {} # Configuration map compilers_mapping = {"c": "CC", "cpp": "CXX", "cuda": "NVCC", "fortran": "FC", "rc": "RC", "ld": "LD", "ar": "AR", "nm": "NM", "ranlib": "RANLIB", "objdump": "OBJDUMP", "strip": "STRIP"} # Compiler definitions by conf - compilers_by_conf = self._conanfile.conf.get("tools.build:compiler_executables", default={}, - check_type=dict) + compilers_by_conf = self._conanfile.conf.get("tools.build:compiler_executables", + default={}, check_type=dict) if compilers_by_conf: for comp, env_var in compilers_mapping.items(): if comp in compilers_by_conf: compiler = compilers_by_conf[comp] # https://github.com/conan-io/conan/issues/13780 compiler = unix_path(self._conanfile, compiler) - extra_env_vars[env_var] = compiler # User/tools ones have precedence + ret[env_var] = compiler # User/tools ones have precedence + # Issue related: https://github.com/conan-io/conan/issues/15486 + if self._is_cross_building and self._conanfile.conf_build: + compilers_build_mapping = ( + self._conanfile.conf_build.get("tools.build:compiler_executables", default={}, + check_type=dict) + ) + if "c" in compilers_build_mapping: + ret["CC_FOR_BUILD"] = compilers_build_mapping["c"] + if "cpp" in compilers_build_mapping: + ret["CXX_FOR_BUILD"] = compilers_build_mapping["cpp"] + return ret + + def _initialize_default_extra_env(self): + """Initialize the default environment variables.""" + # If it's an Android cross-compilation + extra_env_vars = self._resolve_android_cross_compilation() + if not extra_env_vars: + # Normally, these are the most common default flags used by MSVC in Windows + if is_msvc(self._conanfile): + extra_env_vars = {"CC": "cl -nologo", + "CXX": "cl -nologo", + "LD": "link -nologo", + "AR": "lib", + "NM": "dumpbin -symbols", + "OBJDUMP": ":", + "RANLIB": ":", + "STRIP": ":"} + extra_env_vars.update(self._resolve_compilers_mapping_variables()) + # Update the extra_env attribute with all the compiler values for env_var, env_value in extra_env_vars.items(): self.extra_env.define(env_var, env_value) @@ -254,7 +325,7 @@ def generate(self): check_duplicated_generator(self, self._conanfile) # Composing both environments. User extra_env definitions has precedence env_vars = self._environment.vars(self._conanfile) - env_vars.save_script("conanautotoolstoolchain") + env_vars.save_script(GnuToolchain.script_name) # Converts all the arguments into strings args = { "configure_args": cmd_args_to_string(self._dict_to_list(self.configure_args)), diff --git a/test/functional/toolchains/gnu/autotools/test_android.py b/test/functional/toolchains/gnu/autotools/test_android.py index dc0fc430800..665ece68406 100644 --- a/test/functional/toolchains/gnu/autotools/test_android.py +++ b/test/functional/toolchains/gnu/autotools/test_android.py @@ -4,7 +4,6 @@ import pytest -from conan.test.assets.sources import gen_function_cpp, gen_function_h from conan.test.utils.tools import TestClient from test.conftest import tools_locations @@ -17,7 +16,7 @@ @pytest.mark.tool("android_ndk") @pytest.mark.tool("autotools") @pytest.mark.skipif(platform.system() != "Darwin", reason="NDK only installed on MAC") -def test_android_meson_toolchain_cross_compiling(arch, expected_arch): +def test_android_autotools_toolchain_cross_compiling(arch, expected_arch): profile_host = textwrap.dedent(""" include(default) diff --git a/test/functional/toolchains/gnu/test_gnutoolchain_android.py b/test/functional/toolchains/gnu/test_gnutoolchain_android.py new file mode 100644 index 00000000000..b330cbcdf96 --- /dev/null +++ b/test/functional/toolchains/gnu/test_gnutoolchain_android.py @@ -0,0 +1,50 @@ +import os +import platform +import textwrap + +import pytest + +from conan.test.utils.mocks import ConanFileMock +from conan.test.utils.tools import TestClient +from conan.tools.files import replace_in_file +from test.conftest import tools_locations + + +@pytest.mark.parametrize("arch, expected_arch", [('armv8', 'aarch64'), + ('armv7', 'arm'), + ('x86', 'i386'), + ('x86_64', 'x86_64') + ]) +# @pytest.mark.tool("android_ndk") +# @pytest.mark.tool("autotools") +@pytest.mark.skipif(platform.system() != "Darwin", reason="NDK only installed on MAC") +def test_android_gnutoolchain_cross_compiling(arch, expected_arch): + profile_host = textwrap.dedent(""" + include(default) + + [settings] + os = Android + os.api_level = 21 + arch = {arch} + + [conf] + tools.android:ndk_path={ndk_path} + """) + # ndk_path = tools_locations["android_ndk"]["system"]["path"][platform.system()] + ndk_path = "/Users/franchuti/Library/Android/sdk/ndk/24.0.8215888" + profile_host = profile_host.format( + arch=arch, + ndk_path=ndk_path + ) + + client = TestClient(path_with_spaces=False) + # FIXME: Change this when we have gnu_lib as template in the new command + client.run("new autotools_lib -d name=hello -d version=1.0") + replace_in_file(ConanFileMock(), os.path.join(client.current_folder, "conanfile.py"), + "AutotoolsToolchain", "GnuToolchain") + client.save({"profile_host": profile_host}) + client.run("build . --profile:build=default --profile:host=profile_host") + libhello = os.path.join("build-release", "src", ".libs", "libhello.a") + # Check binaries architecture + client.run_command('objdump -f "%s"' % libhello) + assert "architecture: %s" % expected_arch in client.out diff --git a/test/integration/toolchains/gnu/test_gnutoolchain.py b/test/integration/toolchains/gnu/test_gnutoolchain.py index 653599ae65a..0a36f6d820b 100644 --- a/test/integration/toolchains/gnu/test_gnutoolchain.py +++ b/test/integration/toolchains/gnu/test_gnutoolchain.py @@ -37,7 +37,7 @@ def test_extra_flags_via_conf(os_): "profile": profile}) client.run("install . --profile:build=profile --profile:host=profile") toolchain = client.load( - "conanautotoolstoolchain{}".format('.bat' if os_ == "Windows" else '.sh')) + "conangnutoolchain{}".format('.bat' if os_ == "Windows" else '.sh')) if os_ == "Windows": assert 'set "CPPFLAGS=%CPPFLAGS% -DNDEBUG -DDEF1 -DDEF2"' in toolchain assert 'set "CXXFLAGS=%CXXFLAGS% -O3 --flag1 --flag2"' in toolchain @@ -88,7 +88,7 @@ def generate(self): client.save({"conanfile.py": conanfile, "profile": profile}) client.run('install . -pr=./profile') toolchain = client.load( - "conanautotoolstoolchain{}".format('.bat' if platform.system() == "Windows" else '.sh')) + "conangnutoolchain{}".format('.bat' if platform.system() == "Windows" else '.sh')) assert '-Dextra_defines -Ddefines' in toolchain assert 'extra_cxxflags cxxflags' in toolchain @@ -113,7 +113,7 @@ def generate(self): client.save({"conanfile.py": conanfile}) client.run("install . -s:b os=Linux -s:h os=Linux") - content = load(os.path.join(client.current_folder, "conanautotoolstoolchain.sh")) + content = load(os.path.join(client.current_folder, "conangnutoolchain.sh")) assert 'export FOO="BAR"' in content @@ -141,7 +141,7 @@ def test_linker_scripts_via_conf(os_): "profile": profile}) client.run("install . --profile:build=profile --profile:host=profile") toolchain = client.load( - "conanautotoolstoolchain{}".format('.bat' if os_ == "Windows" else '.sh')) + "conangnutoolchain{}".format('.bat' if os_ == "Windows" else '.sh')) if os_ == "Windows": assert 'set "LDFLAGS=%LDFLAGS% --flag5 --flag6 -T\'/linker/scripts/flash.ld\' -T\'/linker/scripts/extra_data.ld\'"' in toolchain else: @@ -255,7 +255,7 @@ class toolRecipe(ConanFile): generators = "GnuToolchain" def build(self): - toolchain = os.path.join(self.generators_folder, "conanautotoolstoolchain.sh") + toolchain = os.path.join(self.generators_folder, "conangnutoolchain.sh") content = load(self, toolchain) assert 'export CC="clang"' in content assert 'export CXX="clang++"' in content @@ -274,11 +274,14 @@ class consumerRecipe(ConanFile): tool_requires = "tool/1.0" def build(self): - toolchain = os.path.join(self.generators_folder, "conanautotoolstoolchain.sh") + toolchain = os.path.join(self.generators_folder, "conangnutoolchain.sh") content = load(self, toolchain) assert 'export CC="gcc"' in content assert 'export CXX="g++"' in content assert 'export RC="windres"' in content + # Issue: https://github.com/conan-io/conan/issues/15486 + assert 'export CC_FOR_BUILD="clang"' in content + assert 'export CXX_FOR_BUILD="clang++"' in content """) client = TestClient() client.save({ @@ -376,7 +379,7 @@ class consumerRecipe(ConanFile): generators = "GnuToolchain" def build(self): - toolchain = os.path.join(self.generators_folder, "conanautotoolstoolchain.bat") + toolchain = os.path.join(self.generators_folder, "conangnutoolchain.bat") content = load(self, toolchain) # Default values and conf ones assert r'set "CC=clang"' in content # conf value has precedence @@ -419,7 +422,7 @@ def generate(self): tc.generate() def build(self): - toolchain = os.path.join(self.generators_folder, "conanautotoolstoolchain.bat") + toolchain = os.path.join(self.generators_folder, "conangnutoolchain.bat") content = load(self, toolchain) # Default values assert r'set "CC=compile clang"' in content