Enable CI Scripts to Run Locally (#35)

This PR introduces some changes to our CI scripts that enable them to be
run locally.

In a nutshell, this PR effectively just adds some prompts that allow
users to provide values for environment variables that are typically
provided by GitHub Actions.

These environment variables ensure that the build and test scripts run
successfully outside of a GH Actions environment.

Breaking down the major changes in this PR by file:

- `tools/rapids-download-conda-from-s3`:
- the changes in this file prompt the user for the necessary env vars
that are needed to identify which artifacts should be downloaded from
downloads.rapids.ai
- `tools/rapids-download-from-s3`:
- the changes in this file ensure that artifacts are obtained from
downloads.rapids.ai (instead of via the `aws` CLI) when not in a CI
environment. VPN access is required to download these artifacts
- `tools/rapids-env-update`:
- sets `sccache` to use read-only mode (see
[here](https://github.com/mozilla/sccache/blob/412f7e00687f388b7fbe14543e307090db558c64/docs/S3.md))
since proper write credentials aren't available to devs running local
builds. sets sensible defaults for other build related vars.
- `tools/rapids-upload-conda-to-s3`:
- exits early before any uploads are attempted since upload should only
happen in GH Actions

The result of these changes is that devs can now volume mount their
locally checked out repository into our CI containers and effortlessly
run build / test scripts the same way CI does:

```sh
docker run \
  --rm \
  -it \
  --gpus all \
  -v $PWD:/cugraph -w /cugraph \
  rapidsai/ci:cuda11.4.1-ubuntu18.04-py3.8

## then inside the container...

./ci/test_cpp.sh
```

This will help developers easily reproduce any build/test failure that
they might encounter in CI with conda packages.

tools/rapids-constants

@@ -1,3 +1,4 @@
 #!/bin/bash
 
 export RAPIDS_DOWNLOADS_BUCKET="rapids-downloads"
+export RAPIDS_DOWNLOADS_DOMAIN="downloads.rapids.ai"

tools/rapids-download-conda-from-s3

@@ -4,7 +4,8 @@
 # Positional Arguments:
 #   1) a string of "cpp" or "python" which determines which conda artifact
 #      should be downloaded
-set -e
+set -euo pipefail
+export RAPIDS_SCRIPT_NAME="rapids-download-conda-from-s3"
 
 pkg_type="$1"
 case "${pkg_type}" in
@@ -18,6 +19,65 @@ case "${pkg_type}" in
     ;;
 esac
 
+
+if [ "${CI:-false}" = "false" ]; then
+  rapids-echo-stderr "Local run detected."
+  rapids-echo-stderr "NVIDIA VPN connectivity is required to download workflow artifacts."
+
+  if [ -z "${RAPIDS_BUILD_TYPE:-}" ]; then
+    {
+      echo ""
+      read -r -p "Enter workflow type (one of: pull-request|branch|nightly): " RAPIDS_BUILD_TYPE
+      export RAPIDS_BUILD_TYPE
+      echo ""
+      echo "Suppress this prompt in the future by setting the 'RAPIDS_BUILD_TYPE' environment variable:"
+      echo "export RAPIDS_BUILD_TYPE=${RAPIDS_BUILD_TYPE}"
+      echo ""
+    } >&2
+  fi
+
+  if [ -z "${RAPIDS_REPOSITORY:-}" ]; then
+    {
+      echo ""
+      read -r -p "Enter org/repository name (e.g. rapidsai/cudf): " RAPIDS_REPOSITORY
+      export RAPIDS_REPOSITORY
+      echo ""
+      echo "Suppress this prompt in the future by setting the 'RAPIDS_REPOSITORY' environment variable:"
+      echo "export RAPIDS_REPOSITORY=${RAPIDS_REPOSITORY}"
+      echo ""
+    } >&2
+  fi
+
+  if [ -z "${RAPIDS_REF_NAME:-}" ]; then
+    {
+      echo ""
+      read -r -p "Enter pull-request number (e.g. 1546): " PR_NUMBER
+      RAPIDS_REF_NAME=pull-request/${PR_NUMBER}
+      export RAPIDS_REF_NAME
+      echo ""
+      echo "Suppress this prompt in the future by setting the 'RAPIDS_REF_NAME' environment variable:"
+      echo "export RAPIDS_REF_NAME=${RAPIDS_REF_NAME}"
+      echo ""
+    } >&2
+  fi
+
+  if [ -z "${RAPIDS_SHA:-}" ]; then
+    {
+      echo ""
+      if RAPIDS_SHA=$(set -e; git rev-parse HEAD 2> /dev/null); then
+        echo "Using HEAD commit for artifact commit hash. Overwrite this by setting the 'RAPIDS_SHA' environment variable:"
+        echo "export RAPIDS_SHA=${RAPIDS_SHA}"
+      else
+        echo "There was a problem acquiring the HEAD commit sha from the current directory."
+        echo "Suppress this prompt in the future by ensuring the current directory is the '${RAPIDS_REPOSITORY}' git repository."
+        read -r -p "Enter full commit sha (e.g. f15776cc449b0c8345f044f713c9c06eb13622f3): " RAPIDS_SHA
+      fi
+      echo ""
+      export RAPIDS_SHA
+    } >&2
+  fi
+fi
+
 # Prepare empty dir to extract tarball to
 channel="/tmp/${pkg_type}_channel"
 

tools/rapids-download-from-s3

@@ -4,7 +4,8 @@
 # Positional Arguments:
 #   1) package name to generate s3 path for
 #   2) location to untar it to
-set -eo pipefail
+set -euo pipefail
+source rapids-constants
 export RAPIDS_SCRIPT_NAME="rapids-download-from-s3"
 
 if [ -z "$1" ] || [ -z "$2" ]; then
@@ -17,8 +18,16 @@ s3_dl_path="$(rapids-s3-path)$1"
 untar_dest="$2"
 mkdir -p "${untar_dest}"
 
-rapids-echo-stderr "Downloading and decompressing ${s3_dl_path} into ${untar_dest}"
-aws s3 cp --only-show-errors "${s3_dl_path}" - | tar xzf - -C "${untar_dest}"
+if [ "${CI:-false}" = "false" ]; then
+  # shellcheck disable=SC2001
+  s3_dl_path=$(echo "${s3_dl_path}" | sed "s|s3://${RAPIDS_DOWNLOADS_BUCKET}|https://${RAPIDS_DOWNLOADS_DOMAIN}|")
+  rapids-echo-stderr "Downloading and decompressing ${s3_dl_path} into ${untar_dest}"
+  wget -qO- "${s3_dl_path}" | tar xzf - -C "${untar_dest}"
+else
+  rapids-echo-stderr "Downloading and decompressing ${s3_dl_path} into ${untar_dest}"
+  aws s3 cp --only-show-errors "${s3_dl_path}" - | tar xzf - -C "${untar_dest}"
+fi
+
 
 # echo path to untarred contents
 echo -n "${untar_dest}"

tools/rapids-echo-stderr

@@ -3,7 +3,7 @@
 # Echo to stderr helper function
 STR=""
 if [[ -n "${RAPIDS_SCRIPT_NAME}" ]]; then
-  STR+="    [${RAPIDS_SCRIPT_NAME}] "
+  STR+="[${RAPIDS_SCRIPT_NAME}] "
 fi
 STR+="$*"
 

tools/rapids-env-update

@@ -2,14 +2,22 @@
 # A utility script that examines environment variables provided
 # by Jenkins to make some environment changes depending on whether
 # a nightly or stable build is occurring.
-set -e
+set -euo pipefail
 
 # Remove nightly channels if build is a release build
 if rapids-is-release-build; then
   conda config --system --remove channels rapidsai-nightly
   conda config --system --remove channels dask/label/dev
 fi
 
+if [ "${CI:-false}" = "false" ]; then
+  # Configure sccache for read-only mode since no credentials
+  # are available in local builds.
+  export SCCACHE_S3_NO_CREDENTIALS=true
+  export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
+  export RAPIDS_BUILD_TYPE=${RAPIDS_BUILD_TYPE:-"pull-request"}
+fi
+
 # If nightly or branch build, append current YYMMDD to version
 if ! rapids-is-release-build; then
   VERSION_SUFFIX=$(date +%y%m%d)

tools/rapids-upload-conda-to-s3

@@ -17,6 +17,12 @@ case "${pkg_type}" in
     ;;
 esac
 
+if [ "${CI:-false}" = "false" ]; then
+  rapids-echo-stderr "Packages from local builds cannot be uploaded to S3."
+  rapids-echo-stderr "Open a PR to have successful builds uploaded."
+  exit 1
+fi
+
 # Prepend `conda_` to PKG_TYPE
 pkg_type="conda_$pkg_type"
 pkg_name="$(rapids-package-name "$pkg_type")"