Add integration tests.

.github/workflows/ci.yml

@@ -321,8 +321,69 @@ jobs:
           LATEST: ${{ needs.check.outputs.is-latest || 'false' }}
         shell: bash
 
+  # we should always have an artifact from a previous build.
+  remote:
+    needs: [shellcheck, test, check]
+    runs-on: ubuntu-latest
+    if: github.actor == 'bors[bot]'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-rust
+
+      - name: LLVM instrument coverage
+        id: remote-cov
+        uses: ./.github/actions/cargo-llvm-cov
+        with:
+          name: integration-remote
+
+      - name: Run Remote Test
+        env:
+          TARGET: aarch64-unknown-linux-gnu
+        run: ./ci/test-remote.sh
+        shell: bash
+
+  bisect:
+    needs: [shellcheck, test, check]
+    runs-on: ubuntu-latest
+    if: github.actor == 'bors[bot]'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-rust
+
+      - name: LLVM instrument coverage
+        id: bisect-cov
+        uses: ./.github/actions/cargo-llvm-cov
+        with:
+          name: integration-bisect
+
+      - name: Run Bisect Test
+        env:
+          TARGET: aarch64-unknown-linux-gnu
+        run: ./ci/test-bisect.sh
+        shell: bash
+
+  docker-in-docker:
+    needs: [shellcheck, test, check]
+    runs-on: ubuntu-latest
+    if: github.actor == 'bors[bot]'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-rust
+
+      - name: LLVM instrument coverage
+        id: docker-in-docker-cov
+        uses: ./.github/actions/cargo-llvm-cov
+        with:
+          name: integration-docker-in-docker
+
+      - name: Run Docker-in-Docker Test
+        env:
+          TARGET: aarch64-unknown-linux-gnu
+        run: ./ci/test-docker-in-docker.sh
+        shell: bash
+
   publish:
-    needs: [build, check]
+    needs: [build, check, fmt, clippy, cargo-deny, remote, bisect, docker-in-docker]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

ci/shared.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+function retry {
+  local tries="${TRIES-5}"
+  local timeout="${TIMEOUT-1}"
+  local try=0
+  local exit_code=0
+
+  while (( try < tries )); do
+    if "${@}"; then
+      return 0
+    else
+      exit_code=$?
+    fi
+
+    sleep "${timeout}"
+    echo "Retrying ..." 1>&2
+    try=$(( try + 1 ))
+    timeout=$(( timeout * 2 ))
+  done
+
+  return ${exit_code}
+}

ci/test-bisect.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091,SC1090
+
+# test to see that custom toolchains work
+
+set -x
+set -eo pipefail
+
+if [[ -z "${TARGET}" ]]; then
+    export TARGET="aarch64-unknown-linux-gnu"
+fi
+
+ci_dir=$(dirname "${BASH_SOURCE[0]}")
+ci_dir=$(realpath "${ci_dir}")
+. "${ci_dir}"/shared.sh
+project_home=$(dirname "${ci_dir}")
+
+main() {
+    local td=
+    local err=
+
+    retry cargo fetch
+    cargo build
+    cargo install cargo-bisect-rustc --debug
+    export CROSS="${project_home}/target/debug/cross"
+
+    td="$(mktemp -d)"
+    git clone --depth 1 https://github.com/cross-rs/rust-cpp-hello-word "${td}"
+
+    pushd "${td}"
+    retry cargo fetch
+    # shellcheck disable=SC2016
+    echo '#!/usr/bin/env bash
+export CROSS_CUSTOM_TOOLCHAIN=1
+exec "${CROSS}" run --target '"${TARGET}" > bisect.sh
+    chmod +x bisect.sh
+
+    if ! err=$(cargo bisect-rustc --script=./bisect.sh --target "${TARGET}" 2>&1 >/dev/null); then
+        if [[ "${err}" != *"does not reproduce the regression"* ]]; then
+            echo "${err}"
+            exit 1
+        fi
+    else
+        echo "should have failed, instead succeeded" 1>&2
+        exit 1
+    fi
+    popd
+
+    rm -rf "${td}"
+}
+
+main

ci/test-docker-in-docker.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1004
+
+# test to see that running docker-in-docker works
+
+set -x
+set -eo pipefail
+
+if [[ -z "${TARGET}" ]]; then
+    export TARGET="aarch64-unknown-linux-gnu"
+fi
+
+source=$(dirname "${BASH_SOURCE[0]}")
+source=$(realpath "${source}")
+home=$(dirname "${source}")
+
+main() {
+    docker run -v "${home}":"${home}" -w "${home}" \
+        --rm -e TARGET -e RUSTFLAGS -e RUST_TEST_THREADS \
+        -e LLVM_PROFILE_FILE -e CARGO_INCREMENTAL \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        docker:18.09-dind sh -c '
+#!/usr/bin/env sh
+set -x
+set -euo pipefail
+
+apk add curl
+curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+source "${HOME}/.cargo/env"
+
+# building on release is slow
+apk add libgcc gcc musl-dev
+cargo test --workspace
+cargo install --path . --force --debug
+
+export CROSS_CONTAINER_IN_CONTAINER=1
+
+apk add git
+td="$(mktemp -d)"
+git clone --depth 1 https://github.com/cross-rs/rust-cpp-hello-word "${td}"
+cd "${td}"
+cross run --target "${TARGET}" --verbose
+
+td="$(mktemp -d)"
+git clone --depth 1 https://github.com/cross-rs/test-workspace "${td}"
+cd "${td}"
+cross build --target "${TARGET}" --workspace \
+    --manifest-path="./workspace/Cargo.toml" --verbose
+cd workspace
+cross build --target "${TARGET}" --workspace --verbose
+cd binary
+cross run --target "${TARGET}" --verbose
+'
+}
+
+main

ci/test-remote.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091,SC1090
+
+# test to see that remote docker support works.
+
+set -x
+set -eo pipefail
+
+export CROSS_REMOTE=1
+if [[ -z "${TARGET}" ]]; then
+    export TARGET="aarch64-unknown-linux-gnu"
+fi
+
+ci_dir=$(dirname "${BASH_SOURCE[0]}")
+ci_dir=$(realpath "${ci_dir}")
+. "${ci_dir}"/shared.sh
+project_home=$(dirname "${ci_dir}")
+
+main() {
+    local err=
+
+    retry cargo fetch
+    cargo build
+    export CROSS="${project_home}/target/debug/cross"
+    export CROSS_UTIL="${project_home}/target/debug/cross-util"
+
+    # if the create volume fails, ensure it exists.
+    if ! err=$("${CROSS_UTIL}" volumes create 2>&1 >/dev/null); then
+        if [[ "${err}" != *"already exists"* ]]; then
+            echo "${err}"
+            exit 1
+        fi
+    fi
+    cross_test_cpp
+    "${CROSS_UTIL}" volumes remove
+
+    # ensure the data volume was removed.
+    cross_test_cpp
+}
+
+cross_test_cpp() {
+    local td=
+    td="$(mktemp -d)"
+
+    git clone --depth 1 https://github.com/cross-rs/rust-cpp-hello-word "${td}"
+
+    pushd "${td}"
+    retry cargo fetch
+    "${CROSS}" run --target "${TARGET}"
+    popd
+
+    rm -rf "${td}"
+}
+
+main

ci/test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC2086
+# shellcheck disable=SC2086,SC1091,SC1090
 
 set -x
 set -euo pipefail
@@ -10,30 +10,9 @@ set -euo pipefail
 
 ci_dir=$(dirname "${BASH_SOURCE[0]}")
 ci_dir=$(realpath "${ci_dir}")
+. "${ci_dir}"/shared.sh
 project_home=$(dirname "${ci_dir}")
 
-function retry {
-  local tries="${TRIES-5}"
-  local timeout="${TIMEOUT-1}"
-  local try=0
-  local exit_code=0
-
-  while (( try < tries )); do
-    if "${@}"; then
-      return 0
-    else
-      exit_code=$?
-    fi
-
-    sleep "${timeout}"
-    echo "Retrying ..." 1>&2
-    try=$(( try + 1 ))
-    timeout=$(( timeout * 2 ))
-  done
-
-  return ${exit_code}
-}
-
 workspace_test() {
   "${CROSS[@]}" build --target "${TARGET}" --workspace "$@" ${CROSS_FLAGS}
   "${CROSS[@]}" run --target "${TARGET}" -p binary "$@" ${CROSS_FLAGS}

src/bin/commands/containers.rs

@@ -482,6 +482,7 @@ pub fn remove_persistent_volume(
     engine: &docker::Engine,
     channel: Option<&str>,
 ) -> cross::Result<()> {
+    // we only need a triple that needs docker: the actual target doesn't matter.
     let msg_info = MessageInfo::create(verbose, quiet, color.as_deref())?;
     let triple = cross::Host::X86_64UnknownLinuxGnu.triple();
     let (_, _, dirs) =