Skip to content

Commit

Permalink
Unify build_native via eng/native/build-commons (#1753)
Browse files Browse the repository at this point in the history
* Unify build_native via eng/native/build-commons
In order to add new platform, architecture, compiler or its newer
version, there are currently multiple places to update, which can be
deduplicated.

This patch unifies:

* directories creation
* prerequisites checking
* `version.c` generation
* native build invocation
* common argument parsing
* building help menu

for various build scripts under coreclr, installer and libraries.

The common entry-point script is now `eng/native/build-commons.sh` and
rest of the scripts under `eng/native` are implementation detail.

Also extracted CMake platform detection in
`eng/native/configureplatform.cmake`, to share with coreclr and
`installer/corehost`.

* Use if [[ cond ]] in all places
  • Loading branch information
am11 authored and janvorli committed Jan 23, 2020
1 parent c6e9145 commit 879e596
Show file tree
Hide file tree
Showing 33 changed files with 1,159 additions and 1,773 deletions.
479 changes: 479 additions & 0 deletions eng/native/build-commons.sh

Large diffs are not rendered by default.

182 changes: 182 additions & 0 deletions eng/native/configureplatform.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
include(CheckPIESupported)

# All code we build should be compiled as position independent
check_pie_supported(OUTPUT_VARIABLE PIE_SUPPORT_OUTPUT LANGUAGES CXX)
if(NOT MSVC AND NOT CMAKE_CXX_LINK_PIE_SUPPORTED)
message(WARNING "PIE is not supported at link time: ${PIE_SUPPORT_OUTPUT}.\n"
"PIE link options will not be passed to linker.")
endif()
set(CMAKE_POSITION_INDEPENDENT_CODE ON)

#----------------------------------------
# Detect and set platform variable names
# - for non-windows build platform & architecture is detected using inbuilt CMAKE variables and cross target component configure
# - for windows we use the passed in parameter to CMAKE to determine build arch
#----------------------------------------
if(CMAKE_SYSTEM_NAME STREQUAL Linux)
set(CLR_CMAKE_HOST_UNIX 1)
if(CLR_CROSS_COMPONENTS_BUILD)
# CMAKE_HOST_SYSTEM_PROCESSOR returns the value of `uname -p` on host.
if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL amd64)
if(CLR_CMAKE_TARGET_ARCH STREQUAL "arm" OR CLR_CMAKE_TARGET_ARCH STREQUAL "armel")
if(CMAKE_CROSSCOMPILING)
set(CLR_CMAKE_HOST_UNIX_X86 1)
else()
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
endif()
else()
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
endif()
elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL i686)
set(CLR_CMAKE_HOST_UNIX_X86 1)
else()
clr_unknown_arch()
endif()
else()
# CMAKE_SYSTEM_PROCESSOR returns the value of `uname -p` on target.
# For the AMD/Intel 64bit architecture two different strings are common.
# Linux and Darwin identify it as "x86_64" while FreeBSD and netbsd uses the
# "amd64" string. Accept either of the two here.
if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64)
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)
set(CLR_CMAKE_HOST_UNIX_ARM 1)
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL arm)
set(CLR_CMAKE_HOST_UNIX_ARM 1)
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64)
set(CLR_CMAKE_HOST_UNIX_ARM64 1)
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL i686)
set(CLR_CMAKE_HOST_UNIX_X86 1)
else()
clr_unknown_arch()
endif()
endif()
set(CLR_CMAKE_HOST_LINUX 1)

# Detect Linux ID
set(LINUX_ID_FILE "/etc/os-release")
if(CMAKE_CROSSCOMPILING)
set(LINUX_ID_FILE "${CMAKE_SYSROOT}${LINUX_ID_FILE}")
endif()

execute_process(
COMMAND bash -c "source ${LINUX_ID_FILE} && echo \$ID"
OUTPUT_VARIABLE CLR_CMAKE_LINUX_ID
OUTPUT_STRIP_TRAILING_WHITESPACE)

if(DEFINED CLR_CMAKE_LINUX_ID)
if(CLR_CMAKE_LINUX_ID STREQUAL tizen)
set(CLR_CMAKE_TARGET_TIZEN_LINUX 1)
elseif(CLR_CMAKE_LINUX_ID STREQUAL alpine)
set(CLR_CMAKE_HOST_ALPINE_LINUX 1)
endif()
endif(DEFINED CLR_CMAKE_LINUX_ID)
endif(CMAKE_SYSTEM_NAME STREQUAL Linux)

if(CMAKE_SYSTEM_NAME STREQUAL Darwin)
set(CLR_CMAKE_HOST_UNIX 1)
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
set(CLR_CMAKE_HOST_DARWIN 1)
set(CMAKE_ASM_COMPILE_OBJECT "${CMAKE_C_COMPILER} <FLAGS> <DEFINES> <INCLUDES> -o <OBJECT> -c <SOURCE>")
endif(CMAKE_SYSTEM_NAME STREQUAL Darwin)

if(CMAKE_SYSTEM_NAME STREQUAL FreeBSD)
set(CLR_CMAKE_HOST_UNIX 1)
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
set(CLR_CMAKE_HOST_FREEBSD 1)
endif(CMAKE_SYSTEM_NAME STREQUAL FreeBSD)

if(CMAKE_SYSTEM_NAME STREQUAL OpenBSD)
set(CLR_CMAKE_HOST_UNIX 1)
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
set(CLR_CMAKE_HOST_OPENBSD 1)
endif(CMAKE_SYSTEM_NAME STREQUAL OpenBSD)

if(CMAKE_SYSTEM_NAME STREQUAL NetBSD)
set(CLR_CMAKE_HOST_UNIX 1)
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
set(CLR_CMAKE_HOST_NETBSD 1)
endif(CMAKE_SYSTEM_NAME STREQUAL NetBSD)

if(CMAKE_SYSTEM_NAME STREQUAL SunOS)
set(CLR_CMAKE_HOST_UNIX 1)
EXECUTE_PROCESS(
COMMAND isainfo -n
OUTPUT_VARIABLE SUNOS_NATIVE_INSTRUCTION_SET
)
if(SUNOS_NATIVE_INSTRUCTION_SET MATCHES "amd64")
set(CLR_CMAKE_HOST_UNIX_AMD64 1)
set(CMAKE_SYSTEM_PROCESSOR "amd64")
else()
clr_unknown_arch()
endif()
set(CLR_CMAKE_HOST_SUNOS 1)
endif(CMAKE_SYSTEM_NAME STREQUAL SunOS)

#--------------------------------------------
# This repo builds two set of binaries
# 1. binaries which execute on target arch machine
# - for such binaries host architecture & target architecture are same
# - eg. coreclr.dll
# 2. binaries which execute on host machine but target another architecture
# - host architecture is different from target architecture
# - eg. crossgen.exe - runs on x64 machine and generates nis targeting arm64
# - for complete list of such binaries refer to file crosscomponents.cmake
#-------------------------------------------------------------
# Set HOST architecture variables
if(CLR_CMAKE_HOST_UNIX_ARM)
set(CLR_CMAKE_HOST_ARCH_ARM 1)
set(CLR_CMAKE_HOST_ARCH "arm")
elseif(CLR_CMAKE_HOST_UNIX_ARM64)
set(CLR_CMAKE_HOST_ARCH_ARM64 1)
set(CLR_CMAKE_HOST_ARCH "arm64")
elseif(CLR_CMAKE_HOST_UNIX_AMD64)
set(CLR_CMAKE_HOST_ARCH_AMD64 1)
set(CLR_CMAKE_HOST_ARCH "x64")
elseif(CLR_CMAKE_HOST_UNIX_X86)
set(CLR_CMAKE_HOST_ARCH_I386 1)
set(CLR_CMAKE_HOST_ARCH "x86")
elseif(WIN32)
# CLR_CMAKE_HOST_ARCH is passed in as param to cmake
if (CLR_CMAKE_HOST_ARCH STREQUAL x64)
set(CLR_CMAKE_HOST_ARCH_AMD64 1)
elseif(CLR_CMAKE_HOST_ARCH STREQUAL x86)
set(CLR_CMAKE_HOST_ARCH_I386 1)
elseif(CLR_CMAKE_HOST_ARCH STREQUAL arm)
set(CLR_CMAKE_HOST_ARCH_ARM 1)
elseif(CLR_CMAKE_HOST_ARCH STREQUAL arm64)
set(CLR_CMAKE_HOST_ARCH_ARM64 1)
else()
clr_unknown_arch()
endif()
endif()

# Set TARGET architecture variables
# Target arch will be a cmake param (optional) for both windows as well as non-windows build
# if target arch is not specified then host & target are same
if(NOT DEFINED CLR_CMAKE_TARGET_ARCH OR CLR_CMAKE_TARGET_ARCH STREQUAL "" )
set(CLR_CMAKE_TARGET_ARCH ${CLR_CMAKE_HOST_ARCH})
endif()

# Set target architecture variables
if (CLR_CMAKE_TARGET_ARCH STREQUAL x64)
set(CLR_CMAKE_TARGET_ARCH_AMD64 1)
elseif(CLR_CMAKE_TARGET_ARCH STREQUAL x86)
set(CLR_CMAKE_TARGET_ARCH_I386 1)
elseif(CLR_CMAKE_TARGET_ARCH STREQUAL arm64)
set(CLR_CMAKE_TARGET_ARCH_ARM64 1)
elseif(CLR_CMAKE_TARGET_ARCH STREQUAL arm)
set(CLR_CMAKE_TARGET_ARCH_ARM 1)
elseif(CLR_CMAKE_TARGET_ARCH STREQUAL armel)
set(CLR_CMAKE_TARGET_ARCH_ARM 1)
set(ARM_SOFTFP 1)
else()
clr_unknown_arch()
endif()

# check if host & target arch combination are valid
if(NOT(CLR_CMAKE_TARGET_ARCH STREQUAL CLR_CMAKE_HOST_ARCH))
if(NOT((CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_ARM64) OR (CLR_CMAKE_HOST_ARCH_I386 AND CLR_CMAKE_TARGET_ARCH_ARM) OR (CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_ARM)))
message(FATAL_ERROR "Invalid host and target arch combination")
endif()
endif()
File renamed without changes.
97 changes: 45 additions & 52 deletions eng/native/gen-buildsys.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,14 @@
# This file invokes cmake and generates the build system for Clang.
#

source="${BASH_SOURCE[0]}"
scriptroot="$( cd -P "$( dirname "$0" )" && pwd )"

# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"

# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"

if [ $# -lt 4 ]
then
if [[ "$#" -lt 4 ]]; then
echo "Usage..."
echo "gen-buildsys.sh <path to top level CMakeLists.txt> <path to intermediate directory> <path to tryrun.cmake directory> <Architecture> <compiler> <compiler major version> <compiler minor version> [build flavor] [ninja] [scan-build] [cmakeargs]"
echo "gen-buildsys.sh <path to top level CMakeLists.txt> <path to tryrun.cmake directory> <path to intermediate directory> <Architecture> <compiler> <compiler major version> <compiler minor version> [build flavor] [ninja] [scan-build] [cmakeargs]"
echo "Specify the path to the top level CMake file."
echo "Specify the path to the directory with tryrun.cmake file."
echo "Specify the path that the build system files are generated in."
echo "Specify the path to the directory with tryrun.cmake file."
echo "Specify the target architecture."
echo "Specify the name of compiler (clang or gcc)."
echo "Specify the major version of compiler."
Expand All @@ -41,7 +29,7 @@ cxxCompiler="$compiler++"
majorVersion="$6"
minorVersion="$7"

if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi
if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi

check_version_exists() {
desired_version=-1
Expand All @@ -58,38 +46,38 @@ check_version_exists() {
echo "$desired_version"
}

if [ -z "$CLR_CC" ]; then
if [[ -z "$CLR_CC" ]]; then

# Set default versions
if [ -z "$majorVersion" ]; then
if [[ -z "$majorVersion" ]]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi
if [[ "$compiler" == "clang" ]]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [[ "$compiler" == "gcc" ]]; then versions=( 9 8 7 6 5 4.9 ); fi

for version in "${versions[@]}"; do
parts=(${version//./ })
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi
if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi
done

if [ -z "$majorVersion" ]; then
if [[ -z "$majorVersion" ]]; then
if command -v "$compiler" > /dev/null; then
if [ "$(uname)" != "Darwin" ]; then
if [[ "$(uname)" != "Darwin" ]]; then
echo "WARN: Specific version of $compiler not found, falling back to use the one in PATH."
fi
export CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")"
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
echo "ERROR: No usable version of $compiler found."
exit 1
fi
else
if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then
if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then
if command -v "$compiler" > /dev/null; then
echo "WARN: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
export CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")"
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
echo "ERROR: Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1
Expand All @@ -99,34 +87,36 @@ if [ -z "$CLR_CC" ]; then
fi
else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [ "$desired_version" = "-1" ]; then
if [[ "$desired_version" == "-1" ]]; then
echo "ERROR: Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1
fi
fi

if [ -z "$CC" ]; then
export CC="$(command -v "$compiler$desired_version")"
export CXX="$(command -v "$cxxCompiler$desired_version")"
if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi
if [[ -z "$CC" ]]; then
CC="$(command -v "$compiler$desired_version")"
CXX="$(command -v "$cxxCompiler$desired_version")"
if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi
fi
else
if [ ! -f "$CLR_CC" ]; then
if [[ ! -f "$CLR_CC" ]]; then
echo "ERROR: CLR_CC is set but path '$CLR_CC' does not exist"
exit 1
fi
export CC="$CLR_CC"
export CXX="$CLR_CXX"
CC="$CLR_CC"
CXX="$CLR_CXX"
fi

if [ -z "$CC" ]; then
if [[ -z "$CC" ]]; then
echo "ERROR: Unable to find $compiler."
exit 1
fi

export CCC_CC="$CC"
export CCC_CXX="$CXX"
export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
CCC_CC="$CC"
CCC_CXX="$CXX"
SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"

export CC CCC_CC CXX CCC_CXX SCAN_BUILD_COMMAND

buildtype=DEBUG
code_coverage=OFF
Expand All @@ -136,11 +126,11 @@ generator="Unix Makefiles"
__UnprocessedCMakeArgs=""

for i in "${@:8}"; do
upperI="$(echo $i | awk '{print toupper($0)}')"
case $upperI in
upperI="$(echo "$i" | awk '{print toupper($0)}')"
case "$upperI" in
# Possible build types are DEBUG, CHECKED, RELEASE, RELWITHDEBINFO.
DEBUG | CHECKED | RELEASE | RELWITHDEBINFO)
buildtype=$upperI
buildtype="$upperI"
;;
NINJA)
generator=Ninja
Expand All @@ -154,26 +144,29 @@ for i in "${@:8}"; do
esac
done

OS=`uname`

cmake_extra_defines=
if [ "$CROSSCOMPILE" == "1" ]; then
if [[ "$CROSSCOMPILE" == "1" ]]; then
if ! [[ -n "$ROOTFS_DIR" ]]; then
echo "ROOTFS_DIR not set for crosscompile"
exit 1
fi
export TARGET_BUILD_ARCH=$build_arch
cmake_extra_defines="$cmake_extra_defines -C $tryrun_dir/tryrun.cmake"

TARGET_BUILD_ARCH="$build_arch"
export TARGET_BUILD_ARCH

if [[ -n "$tryrun_dir" ]]; then
cmake_extra_defines="$cmake_extra_defines -C $tryrun_dir/tryrun.cmake"
fi
cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=$scriptroot/../common/cross/toolchain.cmake"
fi

if [ "$build_arch" == "armel" ]; then
if [[ "$build_arch" == "armel" ]]; then
cmake_extra_defines="$cmake_extra_defines -DARM_SOFTFP=1"
fi

cmake_command=$(command -v cmake)

if [[ "$scan_build" == "ON" && "$SCAN_BUILD_COMMAND" != "" ]]; then
if [[ "$scan_build" == "ON" && -n "$SCAN_BUILD_COMMAND" ]]; then
cmake_command="$SCAN_BUILD_COMMAND $cmake_command"
fi

Expand Down
File renamed without changes.
2 changes: 2 additions & 0 deletions eng/pipelines/installer/jobs/base-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,8 @@ jobs:
# lowercase for RID format. (Detection normally converts, but we're preventing it.)
- name: OutputRidArg
value: /p:OutputRid=linux-musl-${{ parameters.archType }}
- name: _PortableBuild
value: true

- name: BuildArguments
value: >-
Expand Down
Loading

0 comments on commit 879e596

Please sign in to comment.