Improve build scripts

koute · Jun 9, 2021 · c103773 · c103773
1 parent 09d62d0
commit c103773
Show file tree

Hide file tree

Showing 14 changed files with 186 additions and 139 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
-/target/
+target
+target-docker
 **/*.rs.bk
 webui/.cache
 webui/dist

diff --git a/ci/build.sh b/ci/build.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+cd "$(dirname $(readlink -f "$0"))/.."
+
+source ./ci/check_if_nightly.sh
+
+cd preload
+cargo build --release --target=x86_64-unknown-linux-gnu $FEATURES_NIGHTLY
+cd ..
+
+cd cli
+cargo build --release --target=x86_64-unknown-linux-gnu
+cd ..
+
+cd gather
+cargo build --release --target=x86_64-unknown-linux-gnu
+cd ..
diff --git a/ci/build_and_package.sh b/ci/build_and_package.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+cd "$(dirname $(readlink -f "$0"))/.."
+
+./ci/build.sh
+./ci/package.sh
diff --git a/ci/build_for_deployment.sh b/ci/build_for_deployment.sh
diff --git a/ci/check_if_nightly.sh b/ci/check_if_nightly.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set +e
+echo "$(rustc --version)" | grep -q "nightly"
+if [ "$?" = "0" ]; then
+    FEATURES_NIGHTLY="--features nightly"
+else
+    FEATURES_NIGHTLY=""
+fi
+set -e
diff --git a/ci/docker/Dockerfile.hybrid b/ci/docker/Dockerfile.hybrid
@@ -1,8 +1,8 @@
-FROM ubuntu:disco
+FROM ubuntu:focal
 MAINTAINER Jan Bujak (j@exia.io)
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends ca-certificates file gcc g++ git locales make qemu-user
+    DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends ca-certificates file gcc g++ git locales make qemu-user curl yarnpkg
 
 RUN locale-gen en_US.UTF-8 && \
     update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
@@ -40,5 +40,15 @@ RUN mkdir -p \
 
 COPY static/cargo.config /home/user/.cargo/config
 
-USER user
 ENV RUST_BACKTRACE=1
+
+ARG USE_HOST_RUSTC=0
+RUN [ $USE_HOST_RUSTC -eq 1 ] || curl https://static.rust-lang.org/rustup/archive/1.24.1/x86_64-unknown-linux-gnu/rustup-init > rustup-init
+RUN [ $USE_HOST_RUSTC -eq 1 ] || chmod +x rustup-init
+RUN [ $USE_HOST_RUSTC -eq 1 ] || ./rustup-init --profile minimal --default-toolchain nightly-2021-06-08 -y
+RUN [ $USE_HOST_RUSTC -eq 1 ] || run-if-enabled aarch64-unknown-linux-gnu "rustup target add aarch64-unknown-linux-gnu"
+RUN [ $USE_HOST_RUSTC -eq 1 ] || run-if-enabled armv7-unknown-linux-gnueabihf "rustup target add armv7-unknown-linux-gnueabihf"
+RUN [ $USE_HOST_RUSTC -eq 1 ] || run-if-enabled mips64-unknown-linux-gnuabi64 "rustup target add mips64-unknown-linux-gnuabi64"
+
+ARG CARGO_TARGET_DIR=/home/user/cwd/target-docker
+ENV CARGO_TARGET_DIR=$CARGO_TARGET_DIR
diff --git a/ci/docker/run.sh b/ci/docker/run.sh
@@ -2,21 +2,35 @@
 
 set -euo pipefail
 
-CWD="$(pwd)"
 cd "$(dirname $(readlink -f "$0"))"
+CWD="$(pwd)/../.."
 
-TARGET_LIST=$(rustup target list | grep "(installed)" | cut -d " " -f 1)
 EXTRA_ARGS=""
-IS_INTERACTIVE=0
 IMAGE_TAG=crossenv
 
+IS_INTERACTIVE=0
+TARGET_LIST="aarch64-unknown-linux-gnu armv7-unknown-linux-gnueabihf mips64-unknown-linux-gnuabi64"
+USE_HOST_RUSTC=0
+CARGO_TARGET_DIR="/home/user/cwd/target-docker"
+
 set +u
-if [ "$1" == "--interactive" ]; then
-    EXTRA_ARGS="$EXTRA_ARGS --entrypoint /bin/bash -v /:/mnt/host"
-    IS_INTERACTIVE=1
-    IMAGE_TAG=crossenv-interactive
-    shift
-fi
+while true
+do
+    if [ "$1" == "--interactive" ]; then
+        EXTRA_ARGS="$EXTRA_ARGS --entrypoint /bin/bash -v /:/mnt/host"
+        IS_INTERACTIVE=1
+        IMAGE_TAG="$IMAGE_TAG-interactive"
+        shift
+    elif [ "$1" == "--use-host-rustc" ]; then
+        TARGET_LIST=$(rustup target list | grep "(installed)" | cut -d " " -f 1)
+        USE_HOST_RUSTC=1
+        CARGO_TARGET_DIR="/home/user/cwd/target"
+        IMAGE_TAG="$IMAGE_TAG-host-rustc"
+        shift
+    else
+        break
+    fi
+done
 set -u
 
 docker build \
@@ -26,17 +40,19 @@ docker build \
     --build-arg UID="$(id -u)" \
     --build-arg GID="$(id -g)" \
     --build-arg IS_INTERACTIVE="$IS_INTERACTIVE" \
+    --build-arg CARGO_TARGET_DIR="$CARGO_TARGET_DIR" \
+    --build-arg USE_HOST_RUSTC="$USE_HOST_RUSTC" \
     ./
 
+if [ "$USE_HOST_RUSTC" == "1" ]; then
+    EXTRA_ARGS="$EXTRA_ARGS -v $HOME/.rustup:/home/user/.rustup -v $HOME/.cargo/bin:/home/user/.cargo/bin -v $HOME/.cargo/git:/home/user/.cargo/git -v $HOME/.cargo/registry:/home/user/.cargo/registry"
+fi
+
 docker run \
     --rm \
     --tty \
     --interactive \
     $EXTRA_ARGS \
-    -v ~/.rustup:/home/user/.rustup \
-    -v ~/.cargo/bin:/home/user/.cargo/bin \
-    -v ~/.cargo/git:/home/user/.cargo/git \
-    -v ~/.cargo/registry:/home/user/.cargo/registry \
     -v "$CWD:/home/user/cwd" \
     -w /home/user/cwd \
     $IMAGE_TAG \

diff --git a/ci/docker/static/run-if-enabled b/ci/docker/static/run-if-enabled
@@ -5,10 +5,11 @@ set -euo pipefail
 set +e
 echo "$TARGET_LIST" | grep -q "$1"
 R=$?
+shift
 set -e
 
 if [[ "$R" == "0" ]]; then
-    /bin/bash -c "set -euo pipefail ; $2"
+    /bin/bash -c "set -euo pipefail ; $@"
     exit $?
 fi
 

diff --git a/ci/package.sh b/ci/package.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+cd "$(dirname $(readlink -f "$0"))/.."
+
+CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$(dirname $(readlink -f "$0"))/../target}
+
+echo "Packaging for deployment..."
+
+rm -Rf $CARGO_TARGET_DIR/travis-deployment $CARGO_TARGET_DIR/travis-deployment-tmp
+mkdir -p $CARGO_TARGET_DIR/travis-deployment $CARGO_TARGET_DIR/travis-deployment-tmp
+
+cp $CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/libmemory_profiler.so $CARGO_TARGET_DIR/travis-deployment-tmp/
+cp $CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/memory-profiler-cli $CARGO_TARGET_DIR/travis-deployment-tmp/
+cp $CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/memory-profiler-gather $CARGO_TARGET_DIR/travis-deployment-tmp/
+
+cd $CARGO_TARGET_DIR/travis-deployment-tmp
+tar -zcf ../travis-deployment/memory-profiler-x86_64-unknown-linux-gnu.tgz \
+    libmemory_profiler.so \
+    memory-profiler-cli \
+    memory-profiler-gather
+
+echo "Deployment package built!"
diff --git a/ci/run_cross_tests.sh b/ci/run_cross_tests.sh
@@ -3,22 +3,23 @@
 set -euo pipefail
 cd "$(dirname $(readlink -f "$0"))/.."
 
-set +e
-echo "$(rustc --version)" | grep -q "nightly"
-if [ "$?" = "0" ]; then
-    echo "Running on nightly!"
-    EXTRA_ARGS="--features nightly"
-else
-    EXTRA_ARGS=""
-fi
-set -e
+source ./ci/check_if_nightly.sh
 
 export MEMORY_PROFILER_TEST_TARGET=$1
 export MEMORY_PROFILER_TEST_RUNNER=/usr/local/bin/runner
-export CARGO_TARGET_DIR="target/cross"
 
-cargo build --target=$MEMORY_PROFILER_TEST_TARGET -p memory-profiler $EXTRA_ARGS
-MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/debug cargo test -p integration-tests
+cd preload
+cargo build --target=$MEMORY_PROFILER_TEST_TARGET $FEATURES_NIGHTLY
+cd ..
 
-cargo build --target=$MEMORY_PROFILER_TEST_TARGET --release -p memory-profiler
-MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/release cargo test -p integration-tests
+cd integration-tests
+MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/debug cargo test --no-default-features
+cd ..
+
+cd preload
+cargo build --target=$MEMORY_PROFILER_TEST_TARGET $FEATURES_NIGHTLY --release
+cd ..
+
+cd integration-tests
+MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/release cargo test --no-default-features
+cd ..
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
@@ -3,46 +3,29 @@
 set -euo pipefail
 cd "$(dirname $(readlink -f "$0"))/.."
 
-TEST_SUBSET=${TEST_SUBSET:-0}
-TEST_TARGET=${TEST_TARGET:-}
+source ./ci/check_if_nightly.sh
 
 export RUST_BACKTRACE=1
 
-if [[ "$TEST_SUBSET" == 0 || "$TEST_SUBSET" == 1 ]]; then
-    cargo test -p common
-    cargo test -p memory-profiler
-    cargo test -p cli-core
-    cargo test -p server-core
-fi
-
-if [[ "$TEST_SUBSET" == 0 || "$TEST_SUBSET" == 2 ]]; then
-    ./ci/build_for_deployment.sh
-    MEMORY_PROFILER_TEST_PRELOAD_PATH=x86_64-unknown-linux-gnu/release cargo test -p integration-tests
-
-    cargo build -p memory-profiler
-    MEMORY_PROFILER_TEST_PRELOAD_PATH=debug cargo test -p integration-tests
-fi
-
-if [[ "$TEST_SUBSET" == 0 || "$TEST_SUBSET" == 3 ]]; then
-    cargo build --release --target=x86_64-unknown-linux-gnu -p memory-profiler-cli
-
-    if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "aarch64-unknown-linux-gnu" ]]; then
-        rustup target add aarch64-unknown-linux-gnu
-    fi
-    if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "armv7-unknown-linux-gnueabihf" ]]; then
-        rustup target add armv7-unknown-linux-gnueabihf
-    fi
-    if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "mips64-unknown-linux-gnuabi64" ]]; then
-        rustup target add mips64-unknown-linux-gnuabi64
-    fi
-
-    if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "aarch64-unknown-linux-gnu" ]]; then
-        ./ci/docker/run.sh ci/run_cross_tests.sh aarch64-unknown-linux-gnu
-    fi
-    if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "armv7-unknown-linux-gnueabihf" ]]; then
-        ./ci/docker/run.sh ci/run_cross_tests.sh armv7-unknown-linux-gnueabihf
-    fi
-    if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "mips64-unknown-linux-gnuabi64" ]]; then
-        ./ci/docker/run.sh ci/run_cross_tests.sh mips64-unknown-linux-gnuabi64
-    fi
-fi
+cd common
+cargo test --target=x86_64-unknown-linux-gnu
+cd ..
+
+cd preload
+cargo test --target=x86_64-unknown-linux-gnu $FEATURES_NIGHTLY
+cd ..
+
+cd cli-core
+cargo test --target=x86_64-unknown-linux-gnu
+cd ..
+
+cd server-core
+cargo test --target=x86_64-unknown-linux-gnu
+cd ..
+
+./ci/build.sh
+
+ci/run_cross_tests.sh x86_64-unknown-linux-gnu
+ci/run_cross_tests.sh aarch64-unknown-linux-gnu
+ci/run_cross_tests.sh armv7-unknown-linux-gnueabihf
+ci/run_cross_tests.sh mips64-unknown-linux-gnuabi64
diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml
@@ -10,3 +10,7 @@ attohttpc = { version = "0.4", default-features = false }
 serde = "1"
 serde_json = "1"
 serde_derive = "1"
+
+[features]
+default = ["test-wasmtime"]
+test-wasmtime = []