Conda release setup

build.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+# Copyright (c) 2019, NVIDIA CORPORATION.
+
+# cuDF build script
+
+# This script is used to build the component(s) in this repo from
+# source, and can be called with various options to customize the
+# build as needed (see the help output for details)
+
+# Abort script on first error
+set -e
+
+NUMARGS=$#
+ARGS=$*
+
+# NOTE: ensure all dir changes are relative to the location of this
+# script, and that this script resides in the repo dir!
+REPODIR=$(cd $(dirname $0); pwd)
+
+VALIDARGS="clean cuXfilter -v -g -n --allgpuarch -h"
+HELP="$0 [clean] [cuXfilter] [cuDatashader] [-v] [-g] [-n] [-h]
+   clean        - remove all existing build artifacts and configuration (start
+                  over)
+   cuXfilter    - build the cuXfilter library only
+   cuDatashader - build the cuDatashader library only
+   -v           - verbose build mode
+   -g           - build for debug
+   -n           - no install step
+   --allgpuarch - build for all supported GPU architectures
+   -h           - print this text
+"
+CUXFILTER_BUILD_DIR=${REPODIR}/python/cuXfilter/build
+CUDATASHADER_BUILD_DIR=${REPODIR}/../cuDatashader/build
+BUILD_DIRS="${CUXFILTER_BUILD_DIR} #{CUDATASHADER_BUILD_DIR}"
+
+# Set defaults for vars modified by flags to this script
+VERBOSE=""
+BUILD_TYPE=Release
+INSTALL_TARGET=install
+BENCHMARKS=OFF
+BUILD_ALL_GPU_ARCH=0
+
+# Set defaults for vars that may not have been defined externally
+#  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
+#         CONDA_PREFIX, but there is no fallback from there!
+INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}}
+PARALLEL_LEVEL=${PARALLEL_LEVEL:=""}
+
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+if hasArg -h; then
+    echo "${HELP}"
+    exit 0
+fi
+
+# Check for valid usage
+if (( ${NUMARGS} != 0 )); then
+    for a in ${ARGS}; do
+    if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then
+        echo "Invalid option: ${a}"
+        exit 1
+    fi
+    done
+fi
+
+# Process flags
+if hasArg -v; then
+    VERBOSE=1
+fi
+if hasArg -g; then
+    BUILD_TYPE=Debug
+fi
+if hasArg -n; then
+    INSTALL_TARGET=""
+fi
+if hasArg --allgpuarch; then
+    BUILD_ALL_GPU_ARCH=1
+fi
+if hasArg benchmarks; then
+    BENCHMARKS="ON"
+fi
+
+# If clean given, run it prior to any other steps
+if hasArg clean; then
+    # If the dirs to clean are mounted dirs in a container, the
+    # contents should be removed but the mounted dirs will remain.
+    # The find removes all contents but leaves the dirs, the rmdir
+    # attempts to remove the dirs but can fail safely.
+    for bd in ${BUILD_DIRS}; do
+    if [ -d ${bd} ]; then
+        find ${bd} -mindepth 1 -delete
+        rmdir ${bd} || true
+    fi
+    done
+fi
+
+if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then
+    GPU_ARCH="-DGPU_ARCHS="
+    echo "Building for the architecture of the GPU in the system..."
+else
+    GPU_ARCH="-DGPU_ARCHS=ALL"
+    echo "Building for *ALL* supported GPU architectures..."
+fi
+
+################################################################################
+
+# Build and install the cuXfilter Python package
+if (( ${NUMARGS} == 0 )) || hasArg cuXfilter; then
+
+    cd ${REPODIR}/python
+    if [[ ${INSTALL_TARGET} != "" ]]; then
+    python setup.py build_ext --inplace
+    python setup.py install --single-version-externally-managed --record=record.txt
+    else
+    python setup.py build_ext --inplace --library-dir=${LIBCUDF_BUILD_DIR}
+    fi
+fi
+
+# Build and install the cuDatashader Python package
+if (( ${NUMARGS} == 0 )) || hasArg cuDatashader; then
+
+    cd ${REPODIR}/../
+    git clone https://github.com/rapidsai/cuDataShader.git
+    cd ${REPODIR}/../cuDataShader
+    python setup.py install --single-version-externally-managed --record=record.txt
+fi

ci/checks/changelog.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+#########################
+# cuXfilter CHANGELOG Tester #
+#########################
+
+# Checkout master for comparison
+git checkout --quiet master
+
+# Switch back to tip of PR branch
+git checkout --quiet current-pr-branch
+
+# Ignore errors during searching
+set +e
+
+# Get list of modified files between matster and PR branch
+CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md`
+# Check if CHANGELOG has PR ID
+PRNUM=`cat CHANGELOG.md | grep "$PR_ID"`
+RETVAL=0
+
+# Return status of check result
+if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then
+  echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n"
+else
+  echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n"
+  RETVAL=1
+fi
+
+cat << EOF
+  It should be placed under the section for the appropriate release.
+  It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate.
+  It should be formatted as '- PR #<PR number> <Concise human-readable description of the PR's new feature, improvement, or bug fix>'
+    Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs'
+EOF
+
+exit $RETVAL

ci/checks/style.sh

@@ -0,0 +1,38 @@
+# Ignore errors and set path
+set +e
+PATH=/conda/bin:$PATH
+LC_ALL=C.UTF-8
+LANG=C.UTF-8
+
+# Activate common conda env
+source activate gdf
+
+# Run black and get results/return code
+BLACK=`black --check python`
+BLACK_RETVAL=$?
+
+# Run flake8 and get results/return code
+FLAKE=`flake8 python`
+FLAKE_RETVAL=$?
+
+if [ "$BLACK_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: black style check; begin output\n\n"
+  echo -e "$BLACK"
+  echo -e "\n\n>>>> FAILED: black style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: black style check\n\n"
+fi
+
+if [ "$FLAKE_RETVAL" != "0" ]; then
+  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
+  echo -e "$FLAKE"
+  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
+fi
+
+RETVALS=($BLACK_RETVAL $FLAKE_RETVAL)
+IFS=$'\n'
+RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
+
+exit $RETVAL

ci/cpu/build.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+######################################
+# cuDF CPU conda build script for CI #
+######################################
+set -e
+
+# Logger function for build status output
+function logger() {
+  echo -e "\n>>>> $@\n"
+}
+
+# Set path and build parallel level
+export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=4
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# Get latest tag and number of commits since tag
+export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags`
+export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count`
+
+# If nightly build, append current YYMMDD to version
+if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
+  export VERSION_SUFFIX=`date +%y%m%d`
+fi
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+logger "Get env..."
+env
+
+logger "Activate conda env..."
+source activate gdf
+
+logger "Check versions..."
+python --version
+gcc --version
+g++ --version
+conda list
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+################################################################################
+# BUILD - Conda package builds (conda deps: libcudf <- libcudf_cffi <- cudf)
+################################################################################
+
+logger "Build conda pkg for cuxfilter..."
+source ci/cpu/cuxfilter/build_cuXfilter.sh
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+logger "Upload conda pkgs..."
+source ci/cpu/upload_anaconda.sh

ci/cpu/cuXfilter/build_cuXfilter.sh

@@ -0,0 +1,4 @@
+set -e
+
+echo "Building cuXfilter"
+conda build conda/recipes/cuXfilter --python=$PYTHON

ci/cpu/prebuild.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#Upload cudf once per PYTHON
+if [[ "$CUDA" == "9.2" ]]; then
+    export UPLOAD_CUDF=1
+else
+    export UPLOAD_CUDF=0
+fi
+
+#Upload libcudf once per CUDA
+if [[ "$PYTHON" == "3.6" ]]; then
+    export UPLOAD_LIBCUDF=1
+else
+    export UPLOAD_LIBCUDF=0
+fi

ci/cpu/upload_anaconda.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+export  CUXFILTER_FILE=`conda build conda/recipes/cuXfilter --output`
+
+SOURCE_BRANCH=master
+CUDA_REL=${CUDA_VERSION%.*}
+
+# Restrict uploads to master branch
+if [ ${GIT_BRANCH} != ${SOURCE_BRANCH} ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+if [ -z "$MY_UPLOAD_KEY" ]; then
+    echo "No upload key"
+    return 0
+fi
+
+if [ "$UPLOAD_CUXFILTER" == "1" ]; then
+  LABEL_OPTION="--label main --label cuda9.2 --label cuda10.0"
+  echo "LABEL_OPTION=${LABEL_OPTION}"
+
+  test -e ${CUXFILTER_FILE}
+  echo "Upload cuXfilter"
+  echo ${CUXFILTER_FILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${CUXFILTER_FILE}
+
+fi