-
Notifications
You must be signed in to change notification settings - Fork 68
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #66 from AjayThorve/conda_release_setup
Conda release setup
- Loading branch information
Showing
156 changed files
with
7,653 additions
and
842 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,130 @@ | ||
#!/bin/bash | ||
|
||
# Copyright (c) 2019, NVIDIA CORPORATION. | ||
|
||
# cuDF build script | ||
|
||
# This script is used to build the component(s) in this repo from | ||
# source, and can be called with various options to customize the | ||
# build as needed (see the help output for details) | ||
|
||
# Abort script on first error | ||
set -e | ||
|
||
NUMARGS=$# | ||
ARGS=$* | ||
|
||
# NOTE: ensure all dir changes are relative to the location of this | ||
# script, and that this script resides in the repo dir! | ||
REPODIR=$(cd $(dirname $0); pwd) | ||
|
||
VALIDARGS="clean cuxfilter cudatashader -v -g -n --allgpuarch -h" | ||
HELP="$0 [clean] [cuxfilter] [cudatashader] [-v] [-g] [-n] [-h] | ||
clean - remove all existing build artifacts and configuration (start | ||
over) | ||
cuxfilter - build the cuxfilter library only | ||
cudatashader - build the cudatashader library only | ||
-v - verbose build mode | ||
-g - build for debug | ||
-n - no install step | ||
--allgpuarch - build for all supported GPU architectures | ||
-h - print this text | ||
" | ||
CUXFILTER_BUILD_DIR=${REPODIR}/python/cuxfilter/build | ||
CUDATASHADER_BUILD_DIR=${REPODIR}/../cuDatashader/build | ||
BUILD_DIRS="${CUXFILTER_BUILD_DIR} #{CUDATASHADER_BUILD_DIR}" | ||
|
||
# Set defaults for vars modified by flags to this script | ||
VERBOSE="" | ||
BUILD_TYPE=Release | ||
INSTALL_TARGET=install | ||
BENCHMARKS=OFF | ||
BUILD_ALL_GPU_ARCH=0 | ||
|
||
# Set defaults for vars that may not have been defined externally | ||
# FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check | ||
# CONDA_PREFIX, but there is no fallback from there! | ||
INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}} | ||
PARALLEL_LEVEL=${PARALLEL_LEVEL:=""} | ||
|
||
function hasArg { | ||
(( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") | ||
} | ||
|
||
if hasArg -h; then | ||
echo "${HELP}" | ||
exit 0 | ||
fi | ||
|
||
# Check for valid usage | ||
if (( ${NUMARGS} != 0 )); then | ||
for a in ${ARGS}; do | ||
if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then | ||
echo "Invalid option: ${a}" | ||
exit 1 | ||
fi | ||
done | ||
fi | ||
|
||
# Process flags | ||
if hasArg -v; then | ||
VERBOSE=1 | ||
fi | ||
if hasArg -g; then | ||
BUILD_TYPE=Debug | ||
fi | ||
if hasArg -n; then | ||
INSTALL_TARGET="" | ||
fi | ||
if hasArg --allgpuarch; then | ||
BUILD_ALL_GPU_ARCH=1 | ||
fi | ||
if hasArg benchmarks; then | ||
BENCHMARKS="ON" | ||
fi | ||
|
||
# If clean given, run it prior to any other steps | ||
if hasArg clean; then | ||
# If the dirs to clean are mounted dirs in a container, the | ||
# contents should be removed but the mounted dirs will remain. | ||
# The find removes all contents but leaves the dirs, the rmdir | ||
# attempts to remove the dirs but can fail safely. | ||
for bd in ${BUILD_DIRS}; do | ||
if [ -d ${bd} ]; then | ||
find ${bd} -mindepth 1 -delete | ||
rmdir ${bd} || true | ||
fi | ||
done | ||
fi | ||
|
||
if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then | ||
GPU_ARCH="-DGPU_ARCHS=" | ||
echo "Building for the architecture of the GPU in the system..." | ||
else | ||
GPU_ARCH="-DGPU_ARCHS=ALL" | ||
echo "Building for *ALL* supported GPU architectures..." | ||
fi | ||
|
||
################################################################################ | ||
|
||
# Build and install the cuxfilter Python package | ||
if (( ${NUMARGS} == 0 )) || hasArg cuxfilter; then | ||
|
||
cd ${REPODIR}/python | ||
if [[ ${INSTALL_TARGET} != "" ]]; then | ||
python setup.py build_ext --inplace | ||
python setup.py install --single-version-externally-managed --record=record.txt | ||
else | ||
python setup.py build_ext --inplace --library-dir=${LIBCUDF_BUILD_DIR} | ||
fi | ||
fi | ||
|
||
# Build and install the cudatashader Python package | ||
if (( ${NUMARGS} == 0 )) || hasArg cudatashader; then | ||
|
||
cd ${REPODIR}/../ | ||
git clone https://github.com/rapidsai/cuDataShader.git | ||
cd ${REPODIR}/../cuDataShader | ||
python setup.py install --single-version-externally-managed --record=record.txt | ||
cd ${REPODIR} | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
#!/bin/bash | ||
# Copyright (c) 2018, NVIDIA CORPORATION. | ||
######################### | ||
# cuxfilter CHANGELOG Tester # | ||
######################### | ||
|
||
# Checkout master for comparison | ||
git checkout --quiet master | ||
|
||
# Switch back to tip of PR branch | ||
git checkout --quiet current-pr-branch | ||
|
||
# Ignore errors during searching | ||
set +e | ||
|
||
# Get list of modified files between matster and PR branch | ||
CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md` | ||
# Check if CHANGELOG has PR ID | ||
PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` | ||
RETVAL=0 | ||
|
||
# Return status of check result | ||
if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then | ||
echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" | ||
else | ||
echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" | ||
RETVAL=1 | ||
fi | ||
|
||
cat << EOF | ||
It should be placed under the section for the appropriate release. | ||
It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. | ||
It should be formatted as '- PR #<PR number> <Concise human-readable description of the PR's new feature, improvement, or bug fix>' | ||
Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' | ||
EOF | ||
|
||
exit $RETVAL |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
# Ignore errors and set path | ||
set +e | ||
PATH=/conda/bin:$PATH | ||
LC_ALL=C.UTF-8 | ||
LANG=C.UTF-8 | ||
|
||
# Activate common conda env | ||
source activate gdf | ||
|
||
# Run black and get results/return code | ||
BLACK=`black --check python` | ||
BLACK_RETVAL=$? | ||
|
||
# Run flake8 and get results/return code | ||
FLAKE=`flake8 python` | ||
FLAKE_RETVAL=$? | ||
|
||
if [ "$BLACK_RETVAL" != "0" ]; then | ||
echo -e "\n\n>>>> FAILED: black style check; begin output\n\n" | ||
echo -e "$BLACK" | ||
echo -e "\n\n>>>> FAILED: black style check; end output\n\n" | ||
else | ||
echo -e "\n\n>>>> PASSED: black style check\n\n" | ||
fi | ||
|
||
if [ "$FLAKE_RETVAL" != "0" ]; then | ||
echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" | ||
echo -e "$FLAKE" | ||
echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" | ||
else | ||
echo -e "\n\n>>>> PASSED: flake8 style check\n\n" | ||
fi | ||
|
||
RETVALS=($BLACK_RETVAL $FLAKE_RETVAL) | ||
IFS=$'\n' | ||
RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` | ||
|
||
exit $RETVAL | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
#!/bin/bash | ||
# Copyright (c) 2018, NVIDIA CORPORATION. | ||
###################################### | ||
# cuDF CPU conda build script for CI # | ||
###################################### | ||
set -e | ||
|
||
# Logger function for build status output | ||
function logger() { | ||
echo -e "\n>>>> $@\n" | ||
} | ||
|
||
# Set path and build parallel level | ||
export PATH=/conda/bin:/usr/local/cuda/bin:$PATH | ||
export PARALLEL_LEVEL=4 | ||
|
||
# Set home to the job's workspace | ||
export HOME=$WORKSPACE | ||
|
||
# Switch to project root; also root of repo checkout | ||
cd $WORKSPACE | ||
|
||
# Get latest tag and number of commits since tag | ||
export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags` | ||
export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count` | ||
|
||
# If nightly build, append current YYMMDD to version | ||
if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then | ||
export VERSION_SUFFIX=`date +%y%m%d` | ||
fi | ||
|
||
################################################################################ | ||
# SETUP - Check environment | ||
################################################################################ | ||
|
||
logger "Get env..." | ||
env | ||
|
||
logger "Activate conda env..." | ||
source activate gdf | ||
|
||
logger "Check versions..." | ||
python --version | ||
gcc --version | ||
g++ --version | ||
conda list | ||
|
||
# FIX Added to deal with Anancoda SSL verification issues during conda builds | ||
conda config --set ssl_verify False | ||
|
||
################################################################################ | ||
# BUILD - Conda package builds (conda deps: libcudf <- libcudf_cffi <- cudf) | ||
################################################################################ | ||
|
||
logger "Build conda pkg for cuxfilter..." | ||
source ci/cpu/cuxfilter/build_cuxfilter.sh | ||
|
||
################################################################################ | ||
# UPLOAD - Conda packages | ||
################################################################################ | ||
|
||
logger "Upload conda pkgs..." | ||
source ci/cpu/upload_anaconda.sh |
Oops, something went wrong.