Merge pull request #25 from dillon-cullinan/fea-gpuci

[WIP] Integrate cuSpatial with gpuCI

CHANGELOG.md

@@ -7,6 +7,7 @@
 - PR #32 Python API first pass
 - PR #37 Python __init__.py package design
 - PR #38 Add __init__.py empties to resolve issue with PYTHONPATH
+- PR #25 Add gpuCI integration
 
 ## Improvements
 

build.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Copyright (c) 2019, NVIDIA CORPORATION.
+
+# cuSpatial build script
+
+# This script is used to build the component(s) in this repo from
+# source, and can be called with various options to customize the
+# build as needed (see the help output for details)
+
+# Abort script on first error
+set -e
+
+NUMARGS=$#
+ARGS=$*
+
+# NOTE: ensure all dir changes are relative to the location of this
+# script, and that this script resides in the repo dir!
+REPODIR=$(cd $(dirname $0); pwd)
+
+VALIDARGS="clean libcuspatial cuspatial -v -g -n -h"
+HELP="$0 [clean] [libcuspatial] [cuspatial] [-v] [-g] [-n] [-h]
+   clean        - remove all existing build artifacts and configuration (start
+                  over)
+   libcuspatial - build the libcuspatial C++ code only
+   cuspatial    - build the cuspatial Python package
+   -v           - verbose build mode
+   -g           - build for debug
+   -n           - no install step
+   -h           - print this text
+   default action (no args) is to build and install 'libcuspatial' then
+   'cuspatial' targets
+"
+LIBCUSPATIAL_BUILD_DIR=${REPODIR}/cpp/build
+CUSPATIAL_BUILD_DIR=${REPODIR}/python/cuspatial/build
+BUILD_DIRS="${LIBCUSPATIAL_BUILD_DIR} ${CUSPATIAL_BUILD_DIR}"
+
+# Set defaults for vars modified by flags to this script
+VERBOSE=""
+BUILD_TYPE=Release
+INSTALL_TARGET=install
+
+# Set defaults for vars that may not have been defined externally
+#  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
+#         CONDA_PREFIX, but there is no fallback from there!
+INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}}
+PARALLEL_LEVEL=${PARALLEL_LEVEL:=""}
+
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+if hasArg -h; then
+    echo "${HELP}"
+    exit 0
+fi
+
+# Check for valid usage
+if (( ${NUMARGS} != 0 )); then
+    for a in ${ARGS}; do
+    if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then
+        echo "Invalid option: ${a}"
+        exit 1
+    fi
+    done
+fi
+
+# Process flags
+if hasArg -v; then
+    VERBOSE=1
+fi
+if hasArg -g; then
+    BUILD_TYPE=Debug
+fi
+if hasArg -n; then
+    INSTALL_TARGET=""
+fi
+
+# If clean given, run it prior to any other steps
+if hasArg clean; then
+    # If the dirs to clean are mounted dirs in a container, the
+    # contents should be removed but the mounted dirs will remain.
+    # The find removes all contents but leaves the dirs, the rmdir
+    # attempts to remove the dirs but can fail safely.
+    for bd in ${BUILD_DIRS}; do
+    if [ -d ${bd} ]; then
+        find ${bd} -mindepth 1 -delete
+        rmdir ${bd} || true
+    fi
+    done
+fi
+
+################################################################################
+# Configure, build, and install libcuspatial
+if (( ${NUMARGS} == 0 )) || hasArg libcuspatial; then
+
+    mkdir -p ${LIBCUSPATIAL_BUILD_DIR}
+    cd ${LIBCUSPATIAL_BUILD_DIR}
+    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
+          -DCMAKE_CXX11_ABI=ON \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ..
+    make -j ${PARALLEL_LEVEL} install VERBOSE=${VERBOSE}
+fi
+
+# Build and install the cuspatial Python package
+if (( ${NUMARGS} == 0 )) || hasArg cuspatial; then
+
+    cd ${REPODIR}/python/cuspatial
+    if [[ ${INSTALL_TARGET} != "" ]]; then
+        python setup.py build_ext --inplace
+        python setup.py install --single-version-externally-managed --record=record.txt
+    else
+        python setup.py build_ext --inplace --library-dir=${LIBCUSPATIAL_BUILD_DIR}
+    fi
+fi
+

ci/checks/changelog.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+#########################
+# cuSpatial CHANGELOG Tester #
+#########################
+
+# Checkout master for comparison
+git checkout --quiet master
+
+# Switch back to tip of PR branch
+git checkout --quiet current-pr-branch
+
+# Ignore errors during searching
+set +e
+
+# Get list of modified files between matster and PR branch
+CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md`
+# Check if CHANGELOG has PR ID
+PRNUM=`cat CHANGELOG.md | grep "$PR_ID"`
+RETVAL=0
+
+# Return status of check result
+if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then
+  echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n"
+else
+  echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n"
+  RETVAL=1
+fi
+
+cat << EOF
+  It should be placed under the section for the appropriate release.
+  It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate.
+  It should be formatted as '- PR #<PR number> <Concise human-readable description of the PR's new feature, improvement, or bug fix>'
+    Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs'
+
+
+EOF
+
+exit $RETVAL
+

ci/checks/style.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+#####################
+# cuSpatial Style Tester #
+#####################
+
+# Ignore errors and set path
+set +e
+PATH=/conda/bin:$PATH
+LC_ALL=C.UTF-8
+LANG=C.UTF-8
+
+# Activate common conda env
+source activate gdf
+
+# Run isort and get results/return code
+ISORT=`isort --recursive --check-only python`
+ISORT_RETVAL=$?
+
+# Run black and get results/return code
+BLACK=`black --check python`
+BLACK_RETVAL=$?
+
+# Run flake8 and get results/return code
+FLAKE=`flake8 python`
+FLAKE_RETVAL=$?
+
+# Run flake8-cython and get results/return code
+FLAKE_CYTHON=`flake8 --config=python/cuspatial/.flake8.cython`
+FLAKE_CYTHON_RETVAL=$?
+
+# Output results if failure otherwise show pass
+if [ "$ISORT_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
+  echo -e "$ISORT"
+  echo -e "\n\n>>>> FAILED: isort style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: isort style check\n\n"
+fi
+

ci/cpu/build.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+######################################
+# cuSpatial CPU conda build script for CI #
+######################################
+set -e
+
+# Logger function for build status output
+function logger() {
+  echo -e "\n>>>> $@\n"
+}
+
+# Set path and build parallel level
+export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=4
+export CUDF_HOME="${WORKSPACE}/cudf"
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# Get latest tag and number of commits since tag
+export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags`
+export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count`
+export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+logger "Get env..."
+env
+
+logger "Activate conda env..."
+source activate gdf
+
+logger "Check versions..."
+python --version
+gcc --version
+g++ --version
+conda list
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+##########################################################################################
+# BUILD - Conda package builds (conda deps: libcupatial <- cuspatial)
+##########################################################################################
+
+logger "Clone cudf"
+git clone https://github.com/rapidsai/cudf.git -b branch-$MINOR_VERSION $CUDF_HOME
+cd $CUDF_HOME
+git submodule update --init --remote --recursive
+
+logger "Build conda pkg for libcuspatial..."
+cd $WORKSPACE
+source ci/cpu/libcuspatial/build_libcuspatial.sh
+
+logger "Build conda pkg for cuspatial..."
+source ci/cpu/cuspatial/build_cuspatial.sh
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+logger "Upload conda pkgs..."
+source ci/cpu/upload_anaconda.sh
+

ci/cpu/cuspatial/build_cuspatial.sh

@@ -0,0 +1,5 @@
+set -e
+
+echo "Building cuspatial"
+conda build conda/recipes/cuspatial --python=$PYTHON
+

ci/cpu/libcuspatial/build_libcuspatial.sh

@@ -0,0 +1,5 @@
+set -e
+
+echo "Building libcuspatial"
+conda build conda/recipes/libcuspatial
+

ci/cpu/prebuild.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+#Upload cuspatial once per PYTHON
+if [[ "$CUDA" == "9.2" ]]; then
+    export UPLOAD_CUSPATIAL=1
+else
+    export UPLOAD_CUSPATIAL=0
+fi
+
+#Upload libcuspatial once per CUDA
+if [[ "$PYTHON" == "3.6" ]]; then
+    export UPLOAD_LIBCUSPATIAL=1
+else
+    export UPLOAD_LIBCUSPATIAL=0
+fi
+

ci/cpu/upload_anaconda.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+export LIBCUSPATIAL_FILE=`conda build conda/recipes/libcuspatial  --output`
+export CUSPATIAL_FILE=`conda build conda/recipes/cuspatial --python=$PYTHON --output`
+
+SOURCE_BRANCH=master
+CUDA_REL=${CUDA_VERSION%.*}
+
+# Restrict uploads to master branch
+if [ ${GIT_BRANCH} != ${SOURCE_BRANCH} ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+if [ -z "$MY_UPLOAD_KEY" ]; then
+    echo "No upload key"
+    return 0
+fi
+
+if [ "$UPLOAD_LIBCUSPATIAL" == "1" ]; then
+  LABEL_OPTION="--label main --label cuda${CUDA_REL}"
+  echo "LABEL_OPTION=${LABEL_OPTION}"
+
+  test -e ${LIBCUSPATIAL_FILE}
+  echo "Upload libcuspatial"
+  echo ${LIBCUSPATIAL_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${LIBCUSPATIAL_FILE}
+fi
+
+if [ "$UPLOAD_CUSPATIAL" == "1" ]; then
+  LABEL_OPTION="--label main --label cuda9.2 --label cuda10.0"
+  echo "LABEL_OPTION=${LABEL_OPTION}"
+
+  test -e ${CUSPATIAL_FILE}
+  echo "Upload cuspatial"
+  echo ${CUSPATIAL_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${CUSPATIAL_FILE}
+fi
+