New build process script changes & gpuCI enhancements(#338)

This PR includes and supersedes #314 

### New build process script changes
This PR outlines the changes necessary for the new build process (aka Project Flash) to work with `cuspatial`. This PR is necessary to begin shadow testing of the new build process.

The goal is to simultaneously support both the old and new build processes. CI is configured to activate the new process by setting `PROJECT_FLASH=1`. Additionally `BUILD_LIBCUSPATIAL` and `BUILD_CUSPATIAL` will be set to 1 as needed by CI in the new process.

CI will stage build artifacts for python and GPU builds in `ci/artifacts/cuspatial/cpu/` which the build scripts will then consume.

The Project Flash script branch has been manually tested. This PR will test the non-Project Flash branch.

### gpuCI enhancements
- Consolidates some of the CPU gpuCI scripts, reducing number of scripts to maintain
- Standardizes naming of the upload script to upload.sh
- Changes logging to use gpuci_logger where practical
- Updates comments and documentation where necessary
- Enables gpuCI to override environment variables based on node types

Authors:
  - dillon-cullinan <dcullinan92@gmail.com>
  - Raymond Douglass <ray@raydouglass.com>

Approvers:
  - Dillon Cullinan (@dillon-cullinan)
  - AJ Schmidt (@ajschmidt8)

URL: #338

CHANGELOG.md

@@ -4,9 +4,7 @@
 
 ## Improvements
 - PR #297 Upgrade to libcu++ on GitHub.
-
 - PR #332 fix directed_hausdorff_distance's space_offsets name + documentation
-
 - PR #331 Use simplified `rmm::exec_policy`
 
 ## Bug Fixes
@@ -20,7 +18,6 @@
 - PR #321 Improvements to gpuCI scripts
 - PR #325 Convert `cudaStream_t` to `rmm::cuda_stream_view`
 
-
 ## Bug Fixes
 - PR #320 Fix quadtree construction bug: zero out `device_uvector` before `scatter`
 - PR #328 Fix point in polygon test for cudf::gather breaking change

ci/cpu/build.sh

@@ -10,6 +10,9 @@ export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 export CUDF_HOME="${WORKSPACE}/cudf"
 
+export GIT_DESCRIBE=`git describe --tags`
+export MINOR_VERSION=`echo $GIT_DESCRIBE | grep -o -E '([0-9]+\.[0-9]+)'`
+
 # Set home to the job's workspace
 export HOME=$WORKSPACE
 
@@ -28,6 +31,10 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
   export VERSION_SUFFIX=`date +%y%m%d`
 fi
 
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -56,11 +63,24 @@ conda config --set ssl_verify False
 # BUILD - Conda package builds (conda deps: libcupatial <- cuspatial)
 ##########################################################################################
 
-gpuci_logger "Building conda pkd for libcuspatial"
-gpuci_conda_retry build conda/recipes/libcuspatial
+if [ "$BUILD_LIBCUSPATIAL" == '1' ]; then
+  gpuci_logger "Build conda pkg for libcuspatial"
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build conda/recipes/libcuspatial
+  else
+    gpuci_conda_retry build --dirty --no-remove-work-dir conda/recipes/libcuspatial
+  fi
+fi
 
-gpuci_logger "Building conda pkg for cuspatial"
-gpuci_conda_retry build conda/recipes/cuspatial --python=$PYTHON
+if [ "$BUILD_CUSPATIAL" == '1' ]; then
+  gpuci_logger "Build conda pkg for cuspatial"
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build conda/recipes/cuspatial
+  else
+    gpuci_conda_retry build --dirty --no-remove-work-dir \
+        -c $WORKSPACE/ci/artifacts/cuspatial/cpu/conda-bld/ conda/recipes/cuspatial
+  fi
+fi
 
 ################################################################################
 # UPLOAD - Conda packages

ci/cpu/prebuild.sh

@@ -14,3 +14,8 @@ else
     export UPLOAD_LIBCUSPATIAL=0
 fi
 
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    #If project flash is not activate, always build both
+    export BUILD_LIBCUSPATIAL=1
+    export BUILD_CUSPATIAL=1
+fi

ci/cpu/upload.sh

@@ -35,20 +35,18 @@ export CUSPATIAL_FILE=`conda build conda/recipes/cuspatial --python=$PYTHON --ou
 
 gpuci_logger "Starting conda uploads"
 
-if [ "$UPLOAD_LIBCUSPATIAL" == "1" ]; then
+if [[ "$BUILD_LIBCUSPATIAL" == "1" && "$UPLOAD_LIBCUSPATIAL" == "1" ]]; then
   LABEL_OPTION="--label main"
   echo "LABEL_OPTION=${LABEL_OPTION}"
-
   test -e ${LIBCUSPATIAL_FILE}
   echo "Upload libcuspatial"
   echo ${LIBCUSPATIAL_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUSPATIAL_FILE}
 fi
 
-if [ "$UPLOAD_CUSPATIAL" == "1" ]; then
+if [[ "$BUILD_CUSPATIAL" == "1" && "$UPLOAD_CUSPATIAL" == "1" ]]; then
   LABEL_OPTION="--label main"
   echo "LABEL_OPTION=${LABEL_OPTION}"
-
   test -e ${CUSPATIAL_FILE}
   echo "Upload cuspatial"
   echo ${CUSPATIAL_FILE}

ci/docs/build.sh

@@ -36,6 +36,8 @@ gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
+
+gpuci_logger "Show conda info"
 conda info
 conda config --show-sources
 conda list --show-channel-urls

ci/gpu/build.sh

@@ -44,13 +44,15 @@ gpuci_conda_retry install "cudf=${MINOR_VERSION}.*" "cudatoolkit=$CUDA_REL" \
     "rapids-build-env=$MINOR_VERSION.*"
 
 # https://docs.rapids.ai/maintainers/depmgmt/ 
-# conda remove -f rapids-build-env
+# gpuci_conda_retry remove --force rapids-build-env
 # gpuci_conda_retry install "your-pkg=1.0.0"
 
 gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
+
+gpuci_logger "Check conda environment"
 conda info
 conda config --show-sources
 conda list --show-channel-urls
@@ -64,41 +66,95 @@ git clone https://github.com/rapidsai/cudf.git -b branch-$MINOR_VERSION ${CUDF_H
 cd $CUDF_HOME
 git submodule update --init --remote --recursive
 
-################################################################################
-# BUILD - Build libcuspatial and cuSpatial from source
-################################################################################
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    ################################################################################
+    # BUILD - Build libcuspatial and cuSpatial from source
+    ################################################################################
 
-gpuci_logger "Build cuSpatial"
-cd $WORKSPACE
-./build.sh clean libcuspatial cuspatial tests
+    gpuci_logger "Build cuSpatial"
+    cd $WORKSPACE
+    ./build.sh clean libcuspatial cuspatial tests
+
+    ###############################################################################
+    # TEST - Run libcuspatial and cuSpatial Unit Tests
+    ###############################################################################
+
+    if hasArg --skip-tests; then
+        gpuci_logger "Skipping tests"
+    else
+        gpuci_logger "Check GPU usage"
+        nvidia-smi
+
+        gpuci_logger "GoogleTests"
+        cd $WORKSPACE/cpp/build
+
+        for gt in ${WORKSPACE}/cpp/build/gtests/* ; do
+            test_name=$(basename ${gt})
+            echo "Running GoogleTest $test_name"
+            ${gt} --gtest_output=xml:${WORKSPACE}/test-results/
+        done
 
-###############################################################################
-# TEST - Run libcuspatial and cuSpatial Unit Tests
-###############################################################################
+        gpuci_logger "Download/Generate Test Data"
+        #TODO
 
-if hasArg --skip-tests; then
-    gpuci_logger "Skipping tests"
+        gpuci_logger "Test cuSpatial"
+        #TODO
+
+        #Python Unit tests for cuSpatial
+        cd $WORKSPACE/python/cuspatial
+        py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuspatial.xml -v
+    fi
 else
+    export LD_LIBRARY_PATH="$WORKSPACE/ci/artifacts/cuspatial/cpu/conda_work/cpp/build:$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
+
+    TESTRESULTS_DIR=${WORKSPACE}/test-results/
+    mkdir -p ${TESTRESULTS_DIR}
+    SUITEERROR=0
+
     gpuci_logger "Check GPU usage"
     nvidia-smi
 
-    gpuci_logger "GoogleTests"
-    cd $WORKSPACE/cpp/build
-
-    for gt in ${WORKSPACE}/cpp/build/gtests/* ; do
+    gpuci_logger "Running googletests"
+    # run gtests
+    cd $WORKSPACE/ci/artifacts/cuspatial/cpu/conda_work/
+    for gt in cpp/build/gtests/* ; do
         test_name=$(basename ${gt})
         echo "Running GoogleTest $test_name"
-        ${gt} --gtest_output=xml:${WORKSPACE}/test-results/
+        ${gt} --gtest_output=xml:${TESTRESULTS_DIR}
+        EXITCODE=$?
+        if (( ${EXITCODE} != 0 )); then
+            SUITEERROR=${EXITCODE}
+            echo "FAILED: GTest ${gt}"
+        fi
     done
 
+    cd $WORKSPACE/python
+
+    CONDA_FILE=`find $WORKSPACE/ci/artifacts/cuspatial/cpu/conda-bld/ -name "libcuspatial*.tar.bz2"`
+    CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
+    CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
+    gpuci_logger "Installing $CONDA_FILE"
+    conda install -c $WORKSPACE/ci/artifacts/cuspatial/cpu/conda-bld/ "$CONDA_FILE"
+
+    export LIBCUGRAPH_BUILD_DIR="$WORKSPACE/ci/artifacts/cuspatial/cpu/conda_work/build"
+
+    gpuci_logger "Building cuspatial"
+    "$WORKSPACE/build.sh" -v cuspatial
+
+    gpuci_logger "Run pytests"
+    py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuspatial.xml -v
+
+    EXITCODE=$?
+    if (( ${EXITCODE} != 0 )); then
+        SUITEERROR=${EXITCODE}
+        echo "FAILED: 1 or more tests in /cuspatial/python"
+    fi
     gpuci_logger "Download/Generate Test Data"
     #TODO
 
     gpuci_logger "Test cuSpatial"
     #TODO
 
-    #Python Unit tests for cuSpatial
-    cd $WORKSPACE/python/cuspatial
-    py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuspatial.xml -v
+    exit ${SUITEERROR}
 fi
 

conda/recipes/libcuspatial/build.sh

@@ -1,8 +1,8 @@
 # Copyright (c) 2018-2019, NVIDIA CORPORATION.
 
-# This assumes the script is executed from the root of the repo directory
-# show environment
-printenv
 # build cuspatial with verbose output
-cd $WORKSPACE
-./build.sh -v libcuspatial
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    ./build.sh -v libcuspatial
+else
+    ./build.sh -v libcuspatial tests
+fi

conda/recipes/libcuspatial/meta.yaml

@@ -9,7 +9,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
@@ -19,8 +19,8 @@ build:
     - CXX
     - CUDAHOSTCXX
     - PARALLEL_LEVEL
-    - CUDF_HOME
     - VERSION_SUFFIX
+    - PROJECT_FLASH
   run_exports:
     - {{ pin_subpackage("libcuspatial", max_pin="x.x") }}