diff --git a/.gitignore b/.gitignore index d6d5945..cb317f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -/target/ +target +target-docker **/*.rs.bk webui/.cache webui/dist diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..dcbc5ee --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -euo pipefail +cd "$(dirname $(readlink -f "$0"))/.." + +source ./ci/check_if_nightly.sh + +cd preload +cargo build --release --target=x86_64-unknown-linux-gnu $FEATURES_NIGHTLY +cd .. + +cd cli +cargo build --release --target=x86_64-unknown-linux-gnu +cd .. + +cd gather +cargo build --release --target=x86_64-unknown-linux-gnu +cd .. diff --git a/ci/build_and_package.sh b/ci/build_and_package.sh new file mode 100755 index 0000000..a2224a3 --- /dev/null +++ b/ci/build_and_package.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail +cd "$(dirname $(readlink -f "$0"))/.." + +./ci/build.sh +./ci/package.sh diff --git a/ci/build_for_deployment.sh b/ci/build_for_deployment.sh deleted file mode 100755 index c44a57f..0000000 --- a/ci/build_for_deployment.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -set +e -echo "$(rustc --version)" | grep -q "nightly" -if [ "$?" = "0" ]; then - export IS_NIGHTLY=1 -else - export IS_NIGHTLY=0 -fi -set -e - -cd preload -if [ "$IS_NIGHTLY" = "1" ]; then - cargo build --release --target=x86_64-unknown-linux-gnu --features nightly -else - cargo build --release --target=x86_64-unknown-linux-gnu -fi -cd .. - -cargo build --release --target=x86_64-unknown-linux-gnu -p memory-profiler-cli -cargo build --release --target=x86_64-unknown-linux-gnu -p memory-profiler-gather - -echo "Building artifacts for deployment..." - -rm -Rf target/travis-deployment target/travis-deployment-tmp -mkdir -p target/travis-deployment target/travis-deployment-tmp - -cp target/x86_64-unknown-linux-gnu/release/libmemory_profiler.so target/travis-deployment-tmp/ -cp target/x86_64-unknown-linux-gnu/release/memory-profiler-cli target/travis-deployment-tmp/ -cp target/x86_64-unknown-linux-gnu/release/memory-profiler-gather target/travis-deployment-tmp/ - -echo "Packing..." - -cd target/travis-deployment-tmp -tar -zcf ../travis-deployment/memory-profiler-x86_64-unknown-linux-gnu.tgz \ - libmemory_profiler.so \ - memory-profiler-cli \ - memory-profiler-gather - -echo "Deployment package built!" diff --git a/ci/check_if_nightly.sh b/ci/check_if_nightly.sh new file mode 100755 index 0000000..4164717 --- /dev/null +++ b/ci/check_if_nightly.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set +e +echo "$(rustc --version)" | grep -q "nightly" +if [ "$?" = "0" ]; then + FEATURES_NIGHTLY="--features nightly" +else + FEATURES_NIGHTLY="" +fi +set -e diff --git a/ci/docker/Dockerfile.hybrid b/ci/docker/Dockerfile.hybrid index 41b3236..144c81f 100644 --- a/ci/docker/Dockerfile.hybrid +++ b/ci/docker/Dockerfile.hybrid @@ -1,8 +1,8 @@ -FROM ubuntu:disco +FROM ubuntu:focal MAINTAINER Jan Bujak (j@exia.io) RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates file gcc g++ git locales make qemu-user + DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends ca-certificates file gcc g++ git locales make qemu-user curl yarnpkg RUN locale-gen en_US.UTF-8 && \ update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 @@ -40,5 +40,15 @@ RUN mkdir -p \ COPY static/cargo.config /home/user/.cargo/config -USER user ENV RUST_BACKTRACE=1 + +ARG USE_HOST_RUSTC=0 +RUN [ $USE_HOST_RUSTC -eq 1 ] || curl https://static.rust-lang.org/rustup/archive/1.24.1/x86_64-unknown-linux-gnu/rustup-init > rustup-init +RUN [ $USE_HOST_RUSTC -eq 1 ] || chmod +x rustup-init +RUN [ $USE_HOST_RUSTC -eq 1 ] || ./rustup-init --profile minimal --default-toolchain nightly-2021-06-08 -y +RUN [ $USE_HOST_RUSTC -eq 1 ] || run-if-enabled aarch64-unknown-linux-gnu "rustup target add aarch64-unknown-linux-gnu" +RUN [ $USE_HOST_RUSTC -eq 1 ] || run-if-enabled armv7-unknown-linux-gnueabihf "rustup target add armv7-unknown-linux-gnueabihf" +RUN [ $USE_HOST_RUSTC -eq 1 ] || run-if-enabled mips64-unknown-linux-gnuabi64 "rustup target add mips64-unknown-linux-gnuabi64" + +ARG CARGO_TARGET_DIR=/home/user/cwd/target-docker +ENV CARGO_TARGET_DIR=$CARGO_TARGET_DIR diff --git a/ci/docker/run.sh b/ci/docker/run.sh index 79929b5..1fd4195 100755 --- a/ci/docker/run.sh +++ b/ci/docker/run.sh @@ -2,21 +2,35 @@ set -euo pipefail -CWD="$(pwd)" cd "$(dirname $(readlink -f "$0"))" +CWD="$(pwd)/../.." -TARGET_LIST=$(rustup target list | grep "(installed)" | cut -d " " -f 1) EXTRA_ARGS="" -IS_INTERACTIVE=0 IMAGE_TAG=crossenv +IS_INTERACTIVE=0 +TARGET_LIST="aarch64-unknown-linux-gnu armv7-unknown-linux-gnueabihf mips64-unknown-linux-gnuabi64" +USE_HOST_RUSTC=0 +CARGO_TARGET_DIR="/home/user/cwd/target-docker" + set +u -if [ "$1" == "--interactive" ]; then - EXTRA_ARGS="$EXTRA_ARGS --entrypoint /bin/bash -v /:/mnt/host" - IS_INTERACTIVE=1 - IMAGE_TAG=crossenv-interactive - shift -fi +while true +do + if [ "$1" == "--interactive" ]; then + EXTRA_ARGS="$EXTRA_ARGS --entrypoint /bin/bash -v /:/mnt/host" + IS_INTERACTIVE=1 + IMAGE_TAG="$IMAGE_TAG-interactive" + shift + elif [ "$1" == "--use-host-rustc" ]; then + TARGET_LIST=$(rustup target list | grep "(installed)" | cut -d " " -f 1) + USE_HOST_RUSTC=1 + CARGO_TARGET_DIR="/home/user/cwd/target" + IMAGE_TAG="$IMAGE_TAG-host-rustc" + shift + else + break + fi +done set -u docker build \ @@ -26,17 +40,19 @@ docker build \ --build-arg UID="$(id -u)" \ --build-arg GID="$(id -g)" \ --build-arg IS_INTERACTIVE="$IS_INTERACTIVE" \ + --build-arg CARGO_TARGET_DIR="$CARGO_TARGET_DIR" \ + --build-arg USE_HOST_RUSTC="$USE_HOST_RUSTC" \ ./ +if [ "$USE_HOST_RUSTC" == "1" ]; then + EXTRA_ARGS="$EXTRA_ARGS -v $HOME/.rustup:/home/user/.rustup -v $HOME/.cargo/bin:/home/user/.cargo/bin -v $HOME/.cargo/git:/home/user/.cargo/git -v $HOME/.cargo/registry:/home/user/.cargo/registry" +fi + docker run \ --rm \ --tty \ --interactive \ $EXTRA_ARGS \ - -v ~/.rustup:/home/user/.rustup \ - -v ~/.cargo/bin:/home/user/.cargo/bin \ - -v ~/.cargo/git:/home/user/.cargo/git \ - -v ~/.cargo/registry:/home/user/.cargo/registry \ -v "$CWD:/home/user/cwd" \ -w /home/user/cwd \ $IMAGE_TAG \ diff --git a/ci/docker/static/run-if-enabled b/ci/docker/static/run-if-enabled index eca60ed..dea87fc 100644 --- a/ci/docker/static/run-if-enabled +++ b/ci/docker/static/run-if-enabled @@ -5,10 +5,11 @@ set -euo pipefail set +e echo "$TARGET_LIST" | grep -q "$1" R=$? +shift set -e if [[ "$R" == "0" ]]; then - /bin/bash -c "set -euo pipefail ; $2" + /bin/bash -c "set -euo pipefail ; $@" exit $? fi diff --git a/ci/package.sh b/ci/package.sh new file mode 100755 index 0000000..79ab86c --- /dev/null +++ b/ci/package.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail +cd "$(dirname $(readlink -f "$0"))/.." + +CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$(dirname $(readlink -f "$0"))/../target} + +echo "Packaging for deployment..." + +rm -Rf $CARGO_TARGET_DIR/travis-deployment $CARGO_TARGET_DIR/travis-deployment-tmp +mkdir -p $CARGO_TARGET_DIR/travis-deployment $CARGO_TARGET_DIR/travis-deployment-tmp + +cp $CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/libmemory_profiler.so $CARGO_TARGET_DIR/travis-deployment-tmp/ +cp $CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/memory-profiler-cli $CARGO_TARGET_DIR/travis-deployment-tmp/ +cp $CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/memory-profiler-gather $CARGO_TARGET_DIR/travis-deployment-tmp/ + +cd $CARGO_TARGET_DIR/travis-deployment-tmp +tar -zcf ../travis-deployment/memory-profiler-x86_64-unknown-linux-gnu.tgz \ + libmemory_profiler.so \ + memory-profiler-cli \ + memory-profiler-gather + +echo "Deployment package built!" diff --git a/ci/run_cross_tests.sh b/ci/run_cross_tests.sh index 84f200d..f30915f 100755 --- a/ci/run_cross_tests.sh +++ b/ci/run_cross_tests.sh @@ -3,22 +3,23 @@ set -euo pipefail cd "$(dirname $(readlink -f "$0"))/.." -set +e -echo "$(rustc --version)" | grep -q "nightly" -if [ "$?" = "0" ]; then - echo "Running on nightly!" - EXTRA_ARGS="--features nightly" -else - EXTRA_ARGS="" -fi -set -e +source ./ci/check_if_nightly.sh export MEMORY_PROFILER_TEST_TARGET=$1 export MEMORY_PROFILER_TEST_RUNNER=/usr/local/bin/runner -export CARGO_TARGET_DIR="target/cross" -cargo build --target=$MEMORY_PROFILER_TEST_TARGET -p memory-profiler $EXTRA_ARGS -MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/debug cargo test -p integration-tests +cd preload +cargo build --target=$MEMORY_PROFILER_TEST_TARGET $FEATURES_NIGHTLY +cd .. -cargo build --target=$MEMORY_PROFILER_TEST_TARGET --release -p memory-profiler -MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/release cargo test -p integration-tests +cd integration-tests +MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/debug cargo test --no-default-features +cd .. + +cd preload +cargo build --target=$MEMORY_PROFILER_TEST_TARGET $FEATURES_NIGHTLY --release +cd .. + +cd integration-tests +MEMORY_PROFILER_TEST_PRELOAD_PATH=$MEMORY_PROFILER_TEST_TARGET/release cargo test --no-default-features +cd .. diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 3395a1b..fe3d571 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -3,46 +3,29 @@ set -euo pipefail cd "$(dirname $(readlink -f "$0"))/.." -TEST_SUBSET=${TEST_SUBSET:-0} -TEST_TARGET=${TEST_TARGET:-} +source ./ci/check_if_nightly.sh export RUST_BACKTRACE=1 -if [[ "$TEST_SUBSET" == 0 || "$TEST_SUBSET" == 1 ]]; then - cargo test -p common - cargo test -p memory-profiler - cargo test -p cli-core - cargo test -p server-core -fi - -if [[ "$TEST_SUBSET" == 0 || "$TEST_SUBSET" == 2 ]]; then - ./ci/build_for_deployment.sh - MEMORY_PROFILER_TEST_PRELOAD_PATH=x86_64-unknown-linux-gnu/release cargo test -p integration-tests - - cargo build -p memory-profiler - MEMORY_PROFILER_TEST_PRELOAD_PATH=debug cargo test -p integration-tests -fi - -if [[ "$TEST_SUBSET" == 0 || "$TEST_SUBSET" == 3 ]]; then - cargo build --release --target=x86_64-unknown-linux-gnu -p memory-profiler-cli - - if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "aarch64-unknown-linux-gnu" ]]; then - rustup target add aarch64-unknown-linux-gnu - fi - if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "armv7-unknown-linux-gnueabihf" ]]; then - rustup target add armv7-unknown-linux-gnueabihf - fi - if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "mips64-unknown-linux-gnuabi64" ]]; then - rustup target add mips64-unknown-linux-gnuabi64 - fi - - if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "aarch64-unknown-linux-gnu" ]]; then - ./ci/docker/run.sh ci/run_cross_tests.sh aarch64-unknown-linux-gnu - fi - if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "armv7-unknown-linux-gnueabihf" ]]; then - ./ci/docker/run.sh ci/run_cross_tests.sh armv7-unknown-linux-gnueabihf - fi - if [[ "$TEST_TARGET" == "" || "$TEST_TARGET" == "mips64-unknown-linux-gnuabi64" ]]; then - ./ci/docker/run.sh ci/run_cross_tests.sh mips64-unknown-linux-gnuabi64 - fi -fi +cd common +cargo test --target=x86_64-unknown-linux-gnu +cd .. + +cd preload +cargo test --target=x86_64-unknown-linux-gnu $FEATURES_NIGHTLY +cd .. + +cd cli-core +cargo test --target=x86_64-unknown-linux-gnu +cd .. + +cd server-core +cargo test --target=x86_64-unknown-linux-gnu +cd .. + +./ci/build.sh + +ci/run_cross_tests.sh x86_64-unknown-linux-gnu +ci/run_cross_tests.sh aarch64-unknown-linux-gnu +ci/run_cross_tests.sh armv7-unknown-linux-gnueabihf +ci/run_cross_tests.sh mips64-unknown-linux-gnuabi64 diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 543c1ea..9119f41 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -10,3 +10,7 @@ attohttpc = { version = "0.4", default-features = false } serde = "1" serde_json = "1" serde_derive = "1" + +[features] +default = ["test-wasmtime"] +test-wasmtime = [] diff --git a/integration-tests/src/tests.rs b/integration-tests/src/tests.rs index 0423e22..2c0fe69 100644 --- a/integration-tests/src/tests.rs +++ b/integration-tests/src/tests.rs @@ -88,7 +88,7 @@ fn cli_path() -> PathBuf { if let Ok( path ) = std::env::var( "MEMORY_PROFILER_TEST_CLI_PATH" ) { build_root().join( path ).join( "memory-profiler-cli" ) } else { - repository_root().join( "target" ).join( "x86_64-unknown-linux-gnu" ).join( "release" ).join( "memory-profiler-cli" ) + build_root().join( "x86_64-unknown-linux-gnu" ).join( "release" ).join( "memory-profiler-cli" ) } } @@ -226,13 +226,7 @@ fn assert_allocation_backtrace( alloc: &Allocation, expected: &[&str] ) { } fn workdir() -> PathBuf { - let path = repository_root().join( "target" ); - let workdir = if let Some( target ) = target() { - path.join( target ) - } else { - path - }; - + let workdir = build_root(); std::fs::create_dir_all( &workdir ).unwrap(); workdir } @@ -285,15 +279,20 @@ fn get_basename( path: &str ) -> &str { &path[ index_slash..index_dot ] } -fn compile_with_cargo( source: &str ) { +fn compile_with_cargo( source: &str ) -> PathBuf { let source_path = repository_root().join( "integration-tests" ).join( "test-programs" ).join( source ); let args: Vec< &str > = vec![ "build" ]; + let target_dir = build_root().join( "test-programs" ); run( &source_path, "cargo", &args, - EMPTY_ENV + &[ + ("CARGO_TARGET_DIR", target_dir.clone()) + ] ).assert_success(); + + target_dir.join( "debug" ) } fn compile_with_flags( source: &str, extra_flags: &[&str] ) { @@ -939,11 +938,11 @@ fn test_backtrace() { assert!( analysis.response.allocations.iter().any( |alloc| alloc.size == 123456 ) ); } +#[cfg(feature = "test-wasmtime")] #[cfg(target_arch = "x86_64")] #[test] fn test_wasmtime_linking() { - let cwd = repository_root().join( "integration-tests" ).join( "test-programs" ).join( "wasmtime" ).join( "target" ).join( "debug" ); - compile_with_cargo( "wasmtime/linking" ); + let cwd = compile_with_cargo( "wasmtime/linking" ); run_on_target( &cwd, @@ -964,11 +963,11 @@ fn test_wasmtime_linking() { } } +#[cfg(feature = "test-wasmtime")] #[cfg(target_arch = "x86_64")] #[test] fn test_wasmtime_interrupt() { - let cwd = repository_root().join( "integration-tests" ).join( "test-programs" ).join( "wasmtime" ).join( "target" ).join( "debug" ); - compile_with_cargo( "wasmtime/interrupt" ); + let cwd = compile_with_cargo( "wasmtime/interrupt" ); run_on_target( &cwd, diff --git a/server-core/build.rs b/server-core/build.rs index 28c4f4d..8d2d9de 100644 --- a/server-core/build.rs +++ b/server-core/build.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use std::fs::{self, File}; use std::io::{self, Write}; use std::env; +use std::ffi::OsString; fn grab_paths< P: AsRef< Path > >( path: P, output: &mut Vec< PathBuf > ) { let path = path.as_ref(); @@ -21,12 +22,25 @@ fn grab_paths< P: AsRef< Path > >( path: P, output: &mut Vec< PathBuf > ) { } } +fn check_command( command: &str ) -> bool { + match Command::new( command ).args( &[ "--version" ] ).status() { + Err( ref error ) if error.kind() == io::ErrorKind::NotFound => { + false + }, + Err( error ) => { + panic!( "Cannot launch `{}`: {}", command, error ); + }, + Ok( _ ) => true + } +} + fn main() { let src_out_dir: PathBuf = env::var_os( "OUT_DIR" ).expect( "missing OUT_DIR" ).into(); let crate_root: PathBuf = env::var_os( "CARGO_MANIFEST_DIR" ).expect( "missing CARGO_MANIFEST_DIR" ).into(); + let target_dir: PathBuf = env::var_os( "CARGO_TARGET_DIR" ).map( |directory| directory.into() ).unwrap_or( crate_root.join( ".." ).join( "target" ) ); let webui_dir = crate_root.join( ".." ).join( "webui" ); - let webui_out_dir = crate_root.join( ".." ).join( "target" ).join( "webui" ); + let webui_out_dir = target_dir.join( "webui" ); struct Lock { semaphore: Option< semalock::Semalock > @@ -38,7 +52,7 @@ fn main() { } } - let lock_path = crate_root.join( ".." ).join( "target" ).join( ".webui-lock" ); + let lock_path = target_dir.join( ".webui-lock" ); let mut lock = Lock { semaphore: Some( semalock::Semalock::new( &lock_path ).expect( "failed to acquire a semaphore" ) ) }; @@ -46,18 +60,16 @@ fn main() { lock.semaphore.as_mut().unwrap().with( |_| { let _ = fs::remove_dir_all( &webui_out_dir ); - match Command::new( "yarn" ).args( &[ "--version" ] ).status() { - Err( ref error ) if error.kind() == io::ErrorKind::NotFound => { + let mut yarn = "yarn"; + if !check_command( yarn ) { + yarn = "yarnpkg"; + if !check_command( yarn ) { panic!( "Yarn not found; you need to install it before you can build the WebUI" ); - }, - Err( error ) => { - panic!( "Cannot launch Yarn: {}", error ); - }, - Ok( _ ) => {} + } } if !webui_dir.join( "node_modules" ).exists() { - let mut child = Command::new( "yarn" ) + let mut child = Command::new( yarn ) .args( &[ "install" ] ) .current_dir( &webui_dir ) .spawn() @@ -76,8 +88,12 @@ fn main() { assert!( webui_dir.join( "node_modules" ).exists() ); + let mut parcel_build_cmd = OsString::new(); + parcel_build_cmd.push( format!( "$({} bin)/parcel build src/index.html -d ", yarn ) ); + parcel_build_cmd.push( &webui_out_dir ); + let mut child = Command::new( "/bin/sh" ) - .args( &[ "-c", "$(yarn bin)/parcel build src/index.html -d ../target/webui" ] ) + .args( &[ OsString::from( String::from( "-c" ) ), parcel_build_cmd ] ) .current_dir( &webui_dir ) .spawn() .expect( "cannot launch a child process to build the WebUI" );