From 267662113d876dc17dd9a83cbc54007a99a9edef Mon Sep 17 00:00:00 2001 From: Arnav Singh Date: Tue, 28 Jan 2020 15:59:44 -0800 Subject: [PATCH] Add rust-toolchain file instead of always using stable. (#2408) Cherry-pick from master of 463a663af5a80385650cf1deb05dbe5d324985a3 and 8a02d3571e65876e5c86e295f0989714276db37a, except that the toolchain is set to 1.40.0 instead of stable. --- Add rust-toolchain file instead of always using stable. Currently our release branches always use the stable toolchain, just like master. Unfortunately a new stable has the ability to break the build, since it brings new clippy and rustfmt and we enforce clean clippy and formatting in the builds. With this change, we now let rustup read the toolchain from the rust-toolchain file instead of forcing it to use latest stable. The expectation is that when we create release branches, we'll freeze the toolchain version to the latest Rust release at the time. For the Windows ARM32 builds, since we use a custom toolchain instead of rustup, the toolchain version is part of the download URL. --- Fix handling of toolchain-specific Windows ARM32 compiler. - Ensure that compiler zip and expanded directory are resolved relative to the repo root, not current directory. This is needed because the azureiotedge-diagnostics build script changes the current directory when it switches from building amd64 to arm32. - Ensure that compiler zip and expanded directory contain the toolchain name. Previously the existing expanded directory would be used regardless of which toolchain zip was expanded to create it. This is also needed because the build agent is shared for older branches that do not use the toolchain file, so their zip and expanded directories must not conflict. --- builds/checkin/edgelet.yaml | 16 +++-- builds/ci/edgelet.yaml | 4 +- edgelet/build/linux/build.sh | 12 +--- edgelet/build/linux/clippy.sh | 40 ++++-------- edgelet/build/linux/coverage.sh | 62 ------------------ edgelet/build/linux/cross-test.sh | 12 ++-- edgelet/build/linux/cross.sh | 12 ++-- edgelet/build/linux/format.sh | 8 +-- edgelet/build/linux/install.sh | 58 +++++++---------- edgelet/build/linux/package.sh | 8 +-- edgelet/build/linux/test.sh | 12 +--- edgelet/build/windows/build-diagnostics.ps1 | 7 +- edgelet/build/windows/build.ps1 | 8 +-- edgelet/build/windows/clippy.ps1 | 19 +++--- edgelet/build/windows/test.ps1 | 7 +- edgelet/build/windows/util.ps1 | 72 ++++++++++++--------- edgelet/rust-toolchain | 1 + scripts/linux/buildEdgelet.sh | 24 +++---- 18 files changed, 145 insertions(+), 237 deletions(-) delete mode 100755 edgelet/build/linux/coverage.sh create mode 100644 edgelet/rust-toolchain diff --git a/builds/checkin/edgelet.yaml b/builds/checkin/edgelet.yaml index 108f978e191..acaea14d8d9 100644 --- a/builds/checkin/edgelet.yaml +++ b/builds/checkin/edgelet.yaml @@ -134,23 +134,25 @@ jobs: echo "##vso[task.setvariable variable=IOTEDGE_HOMEDIR;]/tmp" echo "##vso[task.setvariable variable=CARGO;]${CARGO_HOME:-"$HOME/.cargo"}/bin/cargo" displayName: Set env variables - - script: edgelet/build/linux/install.sh -t "${RUST_TOOLCHAIN}" + - script: | + echo "$RUST_TOOLCHAIN" > rust-toolchain + displayName: Override Rust toolchain + workingDirectory: edgelet + env: + RUST_TOOLCHAIN: $(rust.toolchain) + - script: edgelet/build/linux/install.sh displayName: Install Rust - script: | curl -L https://github.com/mozilla/grcov/releases/download/v0.5.1/grcov-linux-x86_64.tar.bz2 | tar jxf - curl -L https://raw.github.com/eriwen/lcov-to-cobertura-xml/8c55cd11f80a21e7e46f20f8c81fcde0bf11f5e5/lcov_cobertura/lcov_cobertura.py -o lcov_cobertura.py workingDirectory: edgelet displayName: Install code coverage tools - - script: $CARGO "+${RUST_TOOLCHAIN}" build --verbose + - script: $CARGO build --verbose displayName: Build workingDirectory: edgelet - env: - RUST_TOOLCHAIN: $(rust.toolchain) - - script: $CARGO "+${RUST_TOOLCHAIN}" test --verbose + - script: $CARGO test --verbose displayName: Test workingDirectory: edgelet - env: - RUST_TOOLCHAIN: $(rust.toolchain) - script: | zip -0 target/ccov.zip `find ./target \( -name "*.gc*" \) -print` ./grcov target/ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir "/*" --ignore-dir "*docker-rs*" > target/lcov.info diff --git a/builds/ci/edgelet.yaml b/builds/ci/edgelet.yaml index 44af89c4736..d625ebf5b9f 100644 --- a/builds/ci/edgelet.yaml +++ b/builds/ci/edgelet.yaml @@ -55,12 +55,12 @@ jobs: displayName: armv7-unknown-linux-gnueabihf build inputs: filePath: edgelet/build/linux/cross.sh - arguments: --toolchain armv7-unknown-linux-gnueabihf --release true + arguments: --target armv7-unknown-linux-gnueabihf --release true - task: Bash@3 displayName: armv7-unknown-linux-gnueabihf test inputs: filePath: edgelet/build/linux/cross-test.sh - arguments: --toolchain armv7-unknown-linux-gnueabihf --release true + arguments: --target armv7-unknown-linux-gnueabihf --release true ################################################################################ - job: windows_amd64 diff --git a/edgelet/build/linux/build.sh b/edgelet/build/linux/build.sh index 48ec3bbaf8c..2103849ba0f 100755 --- a/edgelet/build/linux/build.sh +++ b/edgelet/build/linux/build.sh @@ -16,7 +16,6 @@ BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet SCRIPT_NAME=$(basename "$0") CARGO="${CARGO_HOME:-"$HOME/.cargo"}/bin/cargo" -TOOLCHAIN="stable-x86_64-unknown-linux-gnu" RELEASE= ############################################################################### @@ -28,7 +27,6 @@ usage() echo "" echo "options" echo " -h, --help Print this help and exit." - echo " -t, --toolchain Toolchain (default: stable-x86_64-unknown-linux-gnu)" echo " -r, --release Release build? (flag, default: false)" exit 1; } @@ -42,16 +40,12 @@ process_args() for arg in "$@" do if [ $save_next_arg -eq 1 ]; then - TOOLCHAIN="$arg" - save_next_arg=0 - elif [ $save_next_arg -eq 2 ]; then RELEASE="true" save_next_arg=0 else case "$arg" in "-h" | "--help" ) usage;; - "-t" | "--toolchain" ) save_next_arg=1;; - "-r" | "--release" ) save_next_arg=2;; + "-r" | "--release" ) save_next_arg=1;; * ) usage;; esac fi @@ -79,7 +73,7 @@ incremental = false EOF if [[ -z ${RELEASE} ]]; then - cd "$PROJECT_ROOT" && $CARGO "+$TOOLCHAIN" build --all + cd "$PROJECT_ROOT" && $CARGO build --all else - cd "$PROJECT_ROOT" && $CARGO "+$TOOLCHAIN" build --all --release + cd "$PROJECT_ROOT" && $CARGO build --all --release fi diff --git a/edgelet/build/linux/clippy.sh b/edgelet/build/linux/clippy.sh index af2181db28b..4a4d74695d2 100755 --- a/edgelet/build/linux/clippy.sh +++ b/edgelet/build/linux/clippy.sh @@ -15,7 +15,6 @@ DIR=$(cd "$(dirname "$0")" && pwd) BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet SCRIPT_NAME=$(basename "$0") -TOOLCHAIN='stable' RUSTUP="${CARGO_HOME:-"$HOME/.cargo"}/bin/rustup" CARGO="${CARGO_HOME:-"$HOME/.cargo"}/bin/cargo" @@ -37,41 +36,28 @@ function print_help_and_exit() exit 1 } -function run_clippy() -{ - echo "Running clippy..." - (cd $PROJECT_ROOT && $CARGO "+$TOOLCHAIN" clippy --all) - (cd $PROJECT_ROOT && $CARGO "+$TOOLCHAIN" clippy --all --tests) - (cd $PROJECT_ROOT && $CARGO "+$TOOLCHAIN" clippy --all --examples) -} - ############################################################################### # Obtain and validate the options supported by this script ############################################################################### function process_args() { - save_next_arg=0 for arg in "$@" do - if [ $save_next_arg -eq 1 ]; then - save_next_arg=0 - else - case "$arg" in - "-h" | "--help" ) usage;; - * ) usage;; - esac - fi + case "$arg" in + "-h" | "--help" ) usage;; + * ) usage;; + esac done } process_args "$@" -if [[ $USE_DOCKER -eq 1 ]]; then - run_clippy_via_docker -else - echo "Installing $TOOLCHAIN toolchain" - $RUSTUP install "$TOOLCHAIN" - echo "Installing clippy..." - $RUSTUP component add clippy "--toolchain=$TOOLCHAIN" - run_clippy -fi +cd $PROJECT_ROOT + +echo "Installing clippy..." +$RUSTUP component add clippy + +echo "Running clippy..." +$CARGO clippy --all +$CARGO clippy --all --tests +$CARGO clippy --all --examples diff --git a/edgelet/build/linux/coverage.sh b/edgelet/build/linux/coverage.sh deleted file mode 100755 index c302f029476..00000000000 --- a/edgelet/build/linux/coverage.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -############################################################################### -# This script runs code coverage -############################################################################### - -set -e - -############################################################################### -# Define Environment Variables -############################################################################### -# Get directory of running script -DIR=$(cd "$(dirname "$0")" && pwd) - -BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} -PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet -SCRIPT_NAME=$(basename "$0") - -############################################################################### -# Print usage information pertaining to this script and exit -############################################################################### -usage() -{ - echo "$SCRIPT_NAME [options]" - echo "" - echo "options" - echo " -h, --help Print this help and exit." - exit 1; -} - -print_help_and_exit() -{ - echo "Run $SCRIPT_NAME --help for more information." - exit 1 -} - -############################################################################### -# Obtain and validate the options supported by this script -############################################################################### -process_args() -{ - save_next_arg=0 - for arg in "$@" - do - if [ $save_next_arg -eq 1 ]; then - save_next_arg=0 - else - case "$arg" in - "-h" | "--help" ) usage;; - "-t" | "--toolchain" ) save_next_arg=1;; - * ) usage;; - esac - fi - done -} - -process_args "$@" - -echo "Running cargo tarpaulin" -docker run --user "$(id -u)":"$(id -g)" --rm --security-opt seccomp=unconfined -v "$PROJECT_ROOT:/volume" xd009642/tarpaulin cargo tarpaulin --out Xml -mkdir -p "$PROJECT_ROOT/target" -mv "$PROJECT_ROOT/cobertura.xml" target diff --git a/edgelet/build/linux/cross-test.sh b/edgelet/build/linux/cross-test.sh index 15fe0af37e1..8bd091c2e60 100755 --- a/edgelet/build/linux/cross-test.sh +++ b/edgelet/build/linux/cross-test.sh @@ -15,7 +15,7 @@ DIR=$(cd "$(dirname "$0")" && pwd) BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet SCRIPT_NAME=$(basename "$0") -TOOLCHAIN="armv7-unknown-linux-gnueabihf" +TARGET="armv7-unknown-linux-gnueabihf" RELEASE= ############################################################################### @@ -27,7 +27,7 @@ usage() echo "" echo "options" echo " -h, --help Print this help and exit." - echo " -t, --toolchain Toolchain (default: stable-x86_64-unknown-linux-gnu)" + echo " -t, --target Target architecture" echo " -r, --release Release build? (flag, default: false)" exit 1; } @@ -41,7 +41,7 @@ process_args() for arg in "$@" do if [ $save_next_arg -eq 1 ]; then - TOOLCHAIN="$arg" + TARGET="$arg" save_next_arg=0 elif [ $save_next_arg -eq 2 ]; then RELEASE="true" @@ -49,7 +49,7 @@ process_args() else case "$arg" in "-h" | "--help" ) usage;; - "-t" | "--toolchain" ) save_next_arg=1;; + "-t" | "--target" ) save_next_arg=1;; "-r" | "--release" ) save_next_arg=2;; * ) usage;; esac @@ -60,7 +60,7 @@ process_args() process_args "$@" if [[ -z ${RELEASE} ]]; then - cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp cross test --all --target "$TOOLCHAIN" + cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp cross test --all --target "$TARGET" else - cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp cross test --all --release --target "$TOOLCHAIN" + cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp cross test --all --release --target "$TARGET" fi diff --git a/edgelet/build/linux/cross.sh b/edgelet/build/linux/cross.sh index 0ff0539a9d4..d75215540a5 100755 --- a/edgelet/build/linux/cross.sh +++ b/edgelet/build/linux/cross.sh @@ -15,7 +15,7 @@ DIR=$(cd "$(dirname "$0")" && pwd) BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet SCRIPT_NAME=$(basename "$0") -TOOLCHAIN="armv7-unknown-linux-gnueabihf" +TARGET="armv7-unknown-linux-gnueabihf" RELEASE= ############################################################################### @@ -27,7 +27,7 @@ usage() echo "" echo "options" echo " -h, --help Print this help and exit." - echo " -t, --toolchain Toolchain (default: stable-x86_64-unknown-linux-gnu)" + echo " -t, --target Target architecture" echo " -r, --release Release build? (flag, default: false)" exit 1; } @@ -41,7 +41,7 @@ process_args() for arg in "$@" do if [ $save_next_arg -eq 1 ]; then - TOOLCHAIN="$arg" + TARGET="$arg" save_next_arg=0 elif [ $save_next_arg -eq 2 ]; then RELEASE="true" @@ -49,7 +49,7 @@ process_args() else case "$arg" in "-h" | "--help" ) usage;; - "-t" | "--toolchain" ) save_next_arg=1;; + "-t" | "--target" ) save_next_arg=1;; "-r" | "--release" ) save_next_arg=2;; * ) usage;; esac @@ -60,7 +60,7 @@ process_args() process_args "$@" if [[ -z ${RELEASE} ]]; then - cd "$PROJECT_ROOT" && cross build --all --target "$TOOLCHAIN" + cd "$PROJECT_ROOT" && cross build --all --target "$TARGET" else - cd "$PROJECT_ROOT" && cross build --all --release --target "$TOOLCHAIN" + cd "$PROJECT_ROOT" && cross build --all --release --target "$TARGET" fi diff --git a/edgelet/build/linux/format.sh b/edgelet/build/linux/format.sh index 2a91669350a..a9aadaa1bbb 100755 --- a/edgelet/build/linux/format.sh +++ b/edgelet/build/linux/format.sh @@ -57,10 +57,10 @@ process_args() process_args "$@" +cd $PROJECT_ROOT + echo "Installing rustfmt" -echo "$RUSTUP component add rustfmt-preview" -$RUSTUP component add rustfmt-preview +$RUSTUP component add rustfmt echo "Running cargo fmt" -echo "cd $PROJECT_ROOT && $CARGO fmt --all -- --check" -cd $PROJECT_ROOT && $CARGO fmt --all -- --check +$CARGO fmt --all -- --check diff --git a/edgelet/build/linux/install.sh b/edgelet/build/linux/install.sh index 2a65d672379..f1c9cef296e 100755 --- a/edgelet/build/linux/install.sh +++ b/edgelet/build/linux/install.sh @@ -1,7 +1,7 @@ #!/bin/bash ############################################################################### -# This script installs the rust toolchain +# This script installs rustup ############################################################################### set -e @@ -11,7 +11,6 @@ set -e ############################################################################### SCRIPT_NAME=$(basename "$0") RUSTUP="${CARGO_HOME:-"$HOME/.cargo"}/bin/rustup" -TOOLCHAIN="stable" ARM_PACKAGE= BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet @@ -25,35 +24,30 @@ function usage() echo "" echo "options" echo " -h, --help Print this help and exit." - echo " -t, --toolchain Toolchain (default: stable)" echo " -p, --package-arm Add additional dependencies for armhf packaging" exit 1; } ############################################################################### -# Install rust toolchain -# -# @param[1] - toolchain; Rust toolchain to install; Required; -# @param[2] - set_default_toolchain; Boolean to set the toolchain as the -# default toolchain duing its installation; Required; +# Install Rust ############################################################################### -function install_toolchain() +function install_rust() { - toolchain="$1" - set_default_toolchain="$2" - - if [[ $set_default_toolchain == "true" ]]; then - cmd_default_toolchain="--default-toolchain" - else - cmd_default_toolchain="" + if ! command -v "$RUSTUP" >/dev/null; then + echo "Installing rustup" + curl https://sh.rustup.rs -sSf | sh -s -- -y fi - echo "Installing rust toolchain $toolchain" - if command -v "$RUSTUP" >/dev/null; then - $RUSTUP install $toolchain - else - curl https://sh.rustup.rs -sSf | sh -s -- -y $cmd_default_toolchain $toolchain - fi + # Forcibly install the toolchain specified in the rust-toolchain file. + # + # rustup automatically installs a missing toolchain, so it would seem we don't have to do this. + # However, Azure Devops VMs have stable pre-installed, and it's not necessarily latest stable. + # If we let rustup auto-install the toolchain, it would continue to use the old pre-installed stable. + # + # We could check if the toolchain file contains "stable" and conditionally issue a `rustup update stable`, + # but it's simpler to just always `update` whatever toolchain it is. `update` installs the toolchain + # if it hasn't already been installed, so this also works for pinned versions. + rustup update "$(< "$PROJECT_ROOT/rust-toolchain")" } ############################################################################### @@ -61,27 +55,19 @@ function install_toolchain() ############################################################################### function process_args() { - save_next_arg=0 for arg in "$@" do - if [ $save_next_arg -eq 1 ]; then - TOOLCHAIN="$arg" - save_next_arg=0 - else - case "$arg" in - "-h" | "--help" ) usage;; - "-t" | "--toolchain" ) save_next_arg=1;; - "-p" | "--package-arm" ) ARM_PACKAGE=1;; - * ) usage;; - esac - fi + case "$arg" in + "-h" | "--help" ) usage;; + "-p" | "--package-arm" ) ARM_PACKAGE=1;; + * ) usage;; + esac done } process_args "$@" -# install specified toolchain -install_toolchain $TOOLCHAIN true +install_rust # Add trusty repo to get older version of libc6-armhf-cross sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu/ trusty main universe" diff --git a/edgelet/build/linux/package.sh b/edgelet/build/linux/package.sh index 01b62581f20..30f52a5ff1e 100755 --- a/edgelet/build/linux/package.sh +++ b/edgelet/build/linux/package.sh @@ -13,8 +13,6 @@ REVISION="${REVISION:-1}" DEFAULT_VERSION="$(cat "$PROJECT_ROOT/version.txt")" VERSION="${VERSION:-$DEFAULT_VERSION}" -RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-stable}" - CMAKE_ARGS='-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_VERSION=1 -DCMAKE_BUILD_TYPE=Release' CMAKE_ARGS="$CMAKE_ARGS -DBUILD_SHARED=On -Drun_unittests=Off -Duse_default_uuid=On -Duse_emulator=Off -Duse_http=Off" @@ -446,10 +444,9 @@ docker run --rm \ $SETUP_COMMAND - echo 'Installing Rust toolchain $RUST_TOOLCHAIN' && - curl -sSLf https://sh.rustup.rs | sh -s -- -y --default-toolchain '$RUST_TOOLCHAIN' && + echo 'Installing rustup' && + curl -sSLf https://sh.rustup.rs | sh -s -- -y && . ~/.cargo/env && - $RUST_TARGET_COMMAND # libiothsm cd /project/edgelet/target/hsm && @@ -458,6 +455,7 @@ docker run --rm \ # iotedged cd /project/edgelet && + $RUST_TARGET_COMMAND $MAKE_COMMAND " diff --git a/edgelet/build/linux/test.sh b/edgelet/build/linux/test.sh index 32ceb80a76d..19051651e24 100755 --- a/edgelet/build/linux/test.sh +++ b/edgelet/build/linux/test.sh @@ -16,7 +16,6 @@ BUILD_REPOSITORY_LOCALPATH=${BUILD_REPOSITORY_LOCALPATH:-$DIR/../../..} PROJECT_ROOT=${BUILD_REPOSITORY_LOCALPATH}/edgelet SCRIPT_NAME=$(basename "$0") CARGO="${CARGO_HOME:-"$HOME/.cargo"}/bin/cargo" -TOOLCHAIN="stable-x86_64-unknown-linux-gnu" RELEASE= ############################################################################### @@ -28,7 +27,6 @@ usage() echo "" echo "options" echo " -h, --help Print this help and exit." - echo " -t, --toolchain Toolchain (default: stable-x86_64-unknown-linux-gnu)" echo " -r, --release Release build? (flag, default: false)" exit 1; } @@ -42,16 +40,12 @@ process_args() for arg in "$@" do if [ $save_next_arg -eq 1 ]; then - TOOLCHAIN="$arg" - save_next_arg=0 - elif [ $save_next_arg -eq 2 ]; then RELEASE="true" save_next_arg=0 else case "$arg" in "-h" | "--help" ) usage;; - "-t" | "--toolchain" ) save_next_arg=1;; - "-r" | "--release" ) save_next_arg=2;; + "-r" | "--release" ) save_next_arg=1;; * ) usage;; esac fi @@ -61,7 +55,7 @@ process_args() process_args "$@" if [[ -z ${RELEASE} ]]; then - cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp $CARGO "+$TOOLCHAIN" test --all + cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp $CARGO test --all else - cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp $CARGO "+$TOOLCHAIN" test --all --release + cd "$PROJECT_ROOT" && IOTEDGE_HOMEDIR=/tmp $CARGO test --all --release fi diff --git a/edgelet/build/windows/build-diagnostics.ps1 b/edgelet/build/windows/build-diagnostics.ps1 index afff529327a..653c8c04940 100644 --- a/edgelet/build/windows/build-diagnostics.ps1 +++ b/edgelet/build/windows/build-diagnostics.ps1 @@ -43,7 +43,8 @@ for ($i = 0; $i -lt $targetArchs.Length; $i++) { } $cargo = Get-CargoCommand -Arm:$Arm - $ManifestPath = Get-Manifest + + cd (Get-EdgeletFolder) $versionInfoFilePath = Join-Path $env:BUILD_REPOSITORY_LOCALPATH 'versionInfo.json' $env:VERSION = Get-Content $versionInfoFilePath | ConvertFrom-JSON | % version @@ -51,8 +52,8 @@ for ($i = 0; $i -lt $targetArchs.Length; $i++) { $originalRustflags = $env:RUSTFLAGS $env:RUSTFLAGS += ' -C target-feature=+crt-static' - Write-Host "$cargo build -p iotedge-diagnostics $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $BuildConfigOption --manifest-path $ManifestPath" - Invoke-Expression "$cargo build -p iotedge-diagnostics $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $BuildConfigOption --manifest-path $ManifestPath" + Write-Host "$cargo build -p iotedge-diagnostics $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $BuildConfigOption" + Invoke-Expression "$cargo build -p iotedge-diagnostics $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $BuildConfigOption" if ($originalRustflags -eq '') { Remove-Item Env:\RUSTFLAGS } diff --git a/edgelet/build/windows/build.ps1 b/edgelet/build/windows/build.ps1 index 9f847a21519..7098c5d0b6b 100644 --- a/edgelet/build/windows/build.ps1 +++ b/edgelet/build/windows/build.ps1 @@ -23,12 +23,10 @@ if ($Arm) { PatchRustForArm -OpenSSL } -# Run cargo build by specifying the manifest file +cd (Get-EdgeletFolder) -$ManifestPath = Get-Manifest - -Write-Host "$cargo build --all $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $(if ($Release) { '--release' }) --manifest-path $ManifestPath" -Invoke-Expression "$cargo build --all $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $(if ($Release) { '--release' }) --manifest-path $ManifestPath" +Write-Host "$cargo build --all $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $(if ($Release) { '--release' })" +Invoke-Expression "$cargo build --all $(if ($Arm) { '--target thumbv7a-pc-windows-msvc' }) $(if ($Release) { '--release' })" if ($LastExitCode -ne 0) { Throw "cargo build failed with exit code $LastExitCode" diff --git a/edgelet/build/windows/clippy.ps1 b/edgelet/build/windows/clippy.ps1 index 7e6e9b6a22a..669f5f79b7f 100644 --- a/edgelet/build/windows/clippy.ps1 +++ b/edgelet/build/windows/clippy.ps1 @@ -13,7 +13,8 @@ $util = Join-Path -Path $PSScriptRoot -ChildPath "util.ps1" Assert-Rust -Arm:$Arm $cargo = Get-CargoCommand -Arm:$Arm -Write-Host $cargo + +cd (Get-EdgeletFolder) rustup component add clippy @@ -29,26 +30,22 @@ if ($Arm) { PatchRustForArm -OpenSSL } -# Run cargo build by specifying the manifest file - -$ManifestPath = Get-Manifest - -Write-Host "$cargo clippy --all --manifest-path $ManifestPath" -Invoke-Expression "$cargo clippy --all --manifest-path $ManifestPath" +Write-Host "$cargo clippy --all" +Invoke-Expression "$cargo clippy --all" if ($LastExitCode -ne 0) { Throw "cargo clippy --all failed with exit code $LastExitCode" } -Write-Host "$cargo clippy --all --tests --manifest-path $ManifestPath" -Invoke-Expression "$cargo clippy --all --tests --manifest-path $ManifestPath" +Write-Host "$cargo clippy --all --tests" +Invoke-Expression "$cargo clippy --all --tests" if ($LastExitCode -ne 0) { Throw "cargo clippy --all --tests failed with exit code $LastExitCode" } -Write-Host "$cargo clippy --all --examples --manifest-path $ManifestPath" -Invoke-Expression "$cargo clippy --all --examples --manifest-path $ManifestPath" +Write-Host "$cargo clippy --all --examples" +Invoke-Expression "$cargo clippy --all --examples" if ($LastExitCode -ne 0) { Throw "cargo clippy --all --examples failed with exit code $LastExitCode" diff --git a/edgelet/build/windows/test.ps1 b/edgelet/build/windows/test.ps1 index e08251b0470..539d21e913a 100644 --- a/edgelet/build/windows/test.ps1 +++ b/edgelet/build/windows/test.ps1 @@ -13,12 +13,13 @@ Assert-Rust # Run cargo test by specifying the manifest file $cargo = Get-CargoCommand -$ManifestPath = Get-Manifest + +cd (Get-EdgeletFolder) $env:IOTEDGE_HOMEDIR = $env:Temp -Write-Host "$cargo test --all $(if ($Release) { '--release' }) --manifest-path $ManifestPath" -Invoke-Expression "$cargo test --all $(if ($Release) { '--release' }) --manifest-path $ManifestPath" +Write-Host "$cargo test --all $(if ($Release) { '--release' })" +Invoke-Expression "$cargo test --all $(if ($Release) { '--release' })" if ($LastExitCode) { Throw "cargo test failed with exit code $LastExitCode" } diff --git a/edgelet/build/windows/util.ps1 b/edgelet/build/windows/util.ps1 index bddc78e3c55..13333a25233 100644 --- a/edgelet/build/windows/util.ps1 +++ b/edgelet/build/windows/util.ps1 @@ -10,7 +10,9 @@ function Test-RustUp function GetPrivateRustPath { - Join-Path -Path (Get-IotEdgeFolder) -ChildPath 'rust-windows-arm/rust-windows-arm/bin/' + $toolchain = Get-Content -Encoding UTF8 (Join-Path -Path (Get-EdgeletFolder) -ChildPath 'rust-toolchain') + $targetPath = Join-Path -Path (Get-IotEdgeFolder) -ChildPath "rust-windows-arm-$toolchain" + Join-Path -Path $targetPath -ChildPath 'rust-windows-arm/bin/' } function Get-CargoCommand @@ -23,11 +25,8 @@ function Get-CargoCommand # we have private rust arm tool chain downloaded and unzipped to \rust-windows-arm\rust-windows-arm\cargo.exe Join-Path -Path (GetPrivateRustPath) -ChildPath 'cargo.exe' } - elseif (Test-RustUp) { - 'cargo +stable-x86_64-pc-windows-msvc ' - } else { - "$env:USERPROFILE/.cargo/bin/cargo.exe +stable-x86_64-pc-windows-msvc " + 'cargo' } } @@ -57,48 +56,61 @@ function Assert-Rust $ErrorActionPreference = 'Continue' + $toolchain = Get-Content -Encoding UTF8 (Join-Path -Path (Get-EdgeletFolder) -ChildPath 'rust-toolchain') + if ($Arm) { - if (-not (Test-Path 'rust-windows-arm')) { - # if the folder rust-windows-arm exists, we assume the private rust compiler for arm is installed + if (-not (Test-Path (GetPrivateRustPath))) { InstallWinArmPrivateRustCompiler } } - elseif (-not (Test-RustUp)) { - Write-Host "Installing rustup and stable-x86_64-pc-windows-msvc Rust." - Invoke-RestMethod -usebasicparsing 'https://static.rust-lang.org/rustup/dist/i686-pc-windows-gnu/rustup-init.exe' -outfile 'rustup-init.exe' - if ($LastExitCode) - { - Throw "Failed to download rustup with exit code $LastExitCode" - } + else { + if (-not (Test-RustUp)) { + Write-Host "Installing rustup" + Invoke-RestMethod -usebasicparsing 'https://static.rust-lang.org/rustup/dist/i686-pc-windows-gnu/rustup-init.exe' -outfile 'rustup-init.exe' + if ($LastExitCode) + { + Throw "Failed to download rustup with exit code $LastExitCode" + } - Write-Host "Running rustup-init.exe" - ./rustup-init.exe -y --default-toolchain stable-x86_64-pc-windows-msvc - if ($LastExitCode) - { - Throw "Failed to install rust with exit code $LastExitCode" + Write-Host "Running rustup-init.exe" + ./rustup-init.exe -y + if ($LastExitCode) + { + Throw "Failed to install rust with exit code $LastExitCode" + } } - } - else { - Write-Host "Running rustup.exe" - rustup install stable-x86_64-pc-windows-msvc + + Write-Host "Installing / updating $toolchain toolchain" + rustup update $toolchain if ($LastExitCode) { Throw "Failed to install rust with exit code $LastExitCode" } } - + $ErrorActionPreference = 'Stop' } function InstallWinArmPrivateRustCompiler { - $link = 'https://edgebuild.blob.core.windows.net/iotedge-win-arm32v7-tools/rust-windows-arm.zip' - - Write-Host "Downloading $link" $ProgressPreference = 'SilentlyContinue' - Invoke-WebRequest $link -OutFile 'rust-windows-arm.zip' -UseBasicParsing - Write-Host "Extracting $link" - Expand-Archive -Path 'rust-windows-arm.zip' -DestinationPath 'rust-windows-arm' + $toolchain = Get-Content -Encoding UTF8 (Join-Path -Path (Get-EdgeletFolder) -ChildPath 'rust-toolchain') + + $downloadPath = (Join-Path -Path (Get-IotEdgeFolder) -ChildPath "rust-windows-arm-$toolchain.zip") + if (-not (Test-Path $downloadPath)) { + $link = "https://edgebuild.blob.core.windows.net/iotedge-win-arm32v7-tools/rust-windows-arm-$toolchain.zip" + + Write-Host "Downloading $link to $downloadPath" + Invoke-WebRequest $link -OutFile $downloadPath -UseBasicParsing + } + + $targetPath = Join-Path -Path (Get-IotEdgeFolder) -ChildPath "rust-windows-arm-$toolchain" + Write-Host "Extracting $downloadPath to $targetPath" + if (Test-Path $targetPath) { + Remove-Item -Recurse -Force $targetPath + } + Expand-Archive -Path $downloadPath -DestinationPath $targetPath + $ProgressPreference = 'Stop' } diff --git a/edgelet/rust-toolchain b/edgelet/rust-toolchain new file mode 100644 index 00000000000..32b7211cb61 --- /dev/null +++ b/edgelet/rust-toolchain @@ -0,0 +1 @@ +1.40.0 diff --git a/scripts/linux/buildEdgelet.sh b/scripts/linux/buildEdgelet.sh index 3c66d531040..6fc5f85923d 100755 --- a/scripts/linux/buildEdgelet.sh +++ b/scripts/linux/buildEdgelet.sh @@ -11,7 +11,7 @@ set -e # Define Environment Variables ############################################################################### ARCH=$(uname -m) -TOOLCHAIN= +TARGET= STRIP= SCRIPT_NAME=$(basename "$0") PROJECT= @@ -70,7 +70,7 @@ print_args() { echo "Project: $EDGELET_DIR/$PROJECT" echo "Arch: $ARCH" - echo "Toolchain: $TOOLCHAIN" + echo "Target: $TARGET" echo "Image: $DOCKER_IMAGENAME" echo "Namespace: $DOCKER_NAMESPACE" echo "Dockerfile: $DOCKERFILE" @@ -125,12 +125,12 @@ process_args() fi case ${ARCH}_${LIBC} in - amd64_musl) TOOLCHAIN="x86_64-unknown-linux-musl";; - amd64_glibc) TOOLCHAIN="x86_64-unknown-linux-gnu";; - arm32v7_musl) TOOLCHAIN="armv7-unknown-linux-musleabihf";; - arm32v7_glibc) TOOLCHAIN="armv7-unknown-linux-gnueabihf";; - arm64v8_musl) TOOLCHAIN="aarch64-unknown-linux-musl";; - arm64v8_glibc) TOOLCHAIN="aarch64-unknown-linux-gnu";; + amd64_musl) TARGET="x86_64-unknown-linux-musl";; + amd64_glibc) TARGET="x86_64-unknown-linux-gnu";; + arm32v7_musl) TARGET="armv7-unknown-linux-musleabihf";; + arm32v7_glibc) TARGET="armv7-unknown-linux-gnueabihf";; + arm64v8_musl) TARGET="aarch64-unknown-linux-musl";; + arm64v8_glibc) TARGET="aarch64-unknown-linux-gnu";; esac case ${ARCH} in @@ -177,8 +177,8 @@ build_project() # build project with cross cd "$EDGELET_DIR" - execute cross build -p "$PROJECT" --manifest-path=iotedged/Cargo.toml --no-default-features --features runtime-kubernetes "$BUILD_CONFIG_OPTION" --target "$TOOLCHAIN" - execute "$STRIP" "$EDGELET_DIR/target/$TOOLCHAIN/$BUILD_CONFIGURATION/$PROJECT" + execute cross build -p "$PROJECT" --manifest-path=iotedged/Cargo.toml --no-default-features --features runtime-kubernetes "$BUILD_CONFIG_OPTION" --target "$TARGET" + execute "$STRIP" "$EDGELET_DIR/target/$TARGET/$BUILD_CONFIGURATION/$PROJECT" # prepare docker folder local EXE_DOCKER_DIR="$PUBLISH_DIR/$DOCKER_IMAGENAME/docker/linux/$ARCH" @@ -189,10 +189,10 @@ build_project() execute cp "$DOCKERFILE" "$EXE_DOCKERFILE" # copy binaries to publish folder - execute cp "$EDGELET_DIR/target/$TOOLCHAIN/$BUILD_CONFIGURATION/$PROJECT" "$EXE_DOCKER_DIR/" + execute cp "$EDGELET_DIR/target/$TARGET/$BUILD_CONFIGURATION/$PROJECT" "$EXE_DOCKER_DIR/" if [[ ${PROJECT,,} == "iotedged" ]] && [[ ${BUILD_CONFIGURATION} == "release" ]]; then - execute cp "$EDGELET_DIR"/target/"$TOOLCHAIN"/"$BUILD_CONFIGURATION"/build/hsm-sys-*/out/lib/*.so* "$EXE_DOCKER_DIR/" + execute cp "$EDGELET_DIR"/target/"$TARGET"/"$BUILD_CONFIGURATION"/build/hsm-sys-*/out/lib/*.so* "$EXE_DOCKER_DIR/" fi }