-
Notifications
You must be signed in to change notification settings - Fork 68
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Conda release setup #66
Changes from 4 commits
490565b
07d713e
92a4bf3
7be0846
86f276e
80b8e13
57596f7
a62d557
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
#!/bin/bash | ||
|
||
# Copyright (c) 2019, NVIDIA CORPORATION. | ||
|
||
# cuDF build script | ||
|
||
# This script is used to build the component(s) in this repo from | ||
# source, and can be called with various options to customize the | ||
# build as needed (see the help output for details) | ||
|
||
# Abort script on first error | ||
set -e | ||
|
||
NUMARGS=$# | ||
ARGS=$* | ||
|
||
# NOTE: ensure all dir changes are relative to the location of this | ||
# script, and that this script resides in the repo dir! | ||
REPODIR=$(cd $(dirname $0); pwd) | ||
|
||
VALIDARGS="clean cuXfilter -v -g -n --allgpuarch -h" | ||
HELP="$0 [clean] [cuXfilter] [cuDatashader] [-v] [-g] [-n] [-h] | ||
clean - remove all existing build artifacts and configuration (start | ||
over) | ||
cuXfilter - build the cuXfilter library only | ||
cuDatashader - build the cuDatashader library only | ||
-v - verbose build mode | ||
-g - build for debug | ||
-n - no install step | ||
--allgpuarch - build for all supported GPU architectures | ||
-h - print this text | ||
" | ||
CUXFILTER_BUILD_DIR=${REPODIR}/python/cuXfilter/build | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. All lower case directories as well. Should change the python folder name to all lower in the repo as well. |
||
CUDATASHADER_BUILD_DIR=${REPODIR}/../cuDatashader/build | ||
BUILD_DIRS="${CUXFILTER_BUILD_DIR} #{CUDATASHADER_BUILD_DIR}" | ||
|
||
# Set defaults for vars modified by flags to this script | ||
VERBOSE="" | ||
BUILD_TYPE=Release | ||
INSTALL_TARGET=install | ||
BENCHMARKS=OFF | ||
BUILD_ALL_GPU_ARCH=0 | ||
|
||
# Set defaults for vars that may not have been defined externally | ||
# FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check | ||
# CONDA_PREFIX, but there is no fallback from there! | ||
INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}} | ||
PARALLEL_LEVEL=${PARALLEL_LEVEL:=""} | ||
|
||
function hasArg { | ||
(( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") | ||
} | ||
|
||
if hasArg -h; then | ||
echo "${HELP}" | ||
exit 0 | ||
fi | ||
|
||
# Check for valid usage | ||
if (( ${NUMARGS} != 0 )); then | ||
for a in ${ARGS}; do | ||
if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then | ||
echo "Invalid option: ${a}" | ||
exit 1 | ||
fi | ||
done | ||
fi | ||
|
||
# Process flags | ||
if hasArg -v; then | ||
VERBOSE=1 | ||
fi | ||
if hasArg -g; then | ||
BUILD_TYPE=Debug | ||
fi | ||
if hasArg -n; then | ||
INSTALL_TARGET="" | ||
fi | ||
if hasArg --allgpuarch; then | ||
BUILD_ALL_GPU_ARCH=1 | ||
fi | ||
if hasArg benchmarks; then | ||
BENCHMARKS="ON" | ||
fi | ||
|
||
# If clean given, run it prior to any other steps | ||
if hasArg clean; then | ||
# If the dirs to clean are mounted dirs in a container, the | ||
# contents should be removed but the mounted dirs will remain. | ||
# The find removes all contents but leaves the dirs, the rmdir | ||
# attempts to remove the dirs but can fail safely. | ||
for bd in ${BUILD_DIRS}; do | ||
if [ -d ${bd} ]; then | ||
find ${bd} -mindepth 1 -delete | ||
rmdir ${bd} || true | ||
fi | ||
done | ||
fi | ||
|
||
if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then | ||
GPU_ARCH="-DGPU_ARCHS=" | ||
echo "Building for the architecture of the GPU in the system..." | ||
else | ||
GPU_ARCH="-DGPU_ARCHS=ALL" | ||
echo "Building for *ALL* supported GPU architectures..." | ||
fi | ||
|
||
################################################################################ | ||
|
||
# Build and install the cuXfilter Python package | ||
if (( ${NUMARGS} == 0 )) || hasArg cuXfilter; then | ||
|
||
cd ${REPODIR}/python | ||
if [[ ${INSTALL_TARGET} != "" ]]; then | ||
python setup.py build_ext --inplace | ||
python setup.py install --single-version-externally-managed --record=record.txt | ||
else | ||
python setup.py build_ext --inplace --library-dir=${LIBCUDF_BUILD_DIR} | ||
fi | ||
fi | ||
|
||
# Build and install the cuDatashader Python package | ||
if (( ${NUMARGS} == 0 )) || hasArg cuDatashader; then | ||
|
||
cd ${REPODIR}/../ | ||
git clone https://github.com/rapidsai/cuDataShader.git | ||
cd ${REPODIR}/../cuDataShader | ||
python setup.py install --single-version-externally-managed --record=record.txt | ||
Comment on lines
+127
to
+128
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would |
||
fi |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
#!/bin/bash | ||
# Copyright (c) 2018, NVIDIA CORPORATION. | ||
######################### | ||
# cuXfilter CHANGELOG Tester # | ||
######################### | ||
|
||
# Checkout master for comparison | ||
git checkout --quiet master | ||
|
||
# Switch back to tip of PR branch | ||
git checkout --quiet current-pr-branch | ||
|
||
# Ignore errors during searching | ||
set +e | ||
|
||
# Get list of modified files between matster and PR branch | ||
CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md` | ||
# Check if CHANGELOG has PR ID | ||
PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` | ||
RETVAL=0 | ||
|
||
# Return status of check result | ||
if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then | ||
echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" | ||
else | ||
echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" | ||
RETVAL=1 | ||
fi | ||
|
||
cat << EOF | ||
It should be placed under the section for the appropriate release. | ||
It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. | ||
It should be formatted as '- PR #<PR number> <Concise human-readable description of the PR's new feature, improvement, or bug fix>' | ||
Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' | ||
EOF | ||
|
||
exit $RETVAL |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
# Ignore errors and set path | ||
set +e | ||
PATH=/conda/bin:$PATH | ||
LC_ALL=C.UTF-8 | ||
LANG=C.UTF-8 | ||
|
||
# Activate common conda env | ||
source activate gdf | ||
|
||
# Run black and get results/return code | ||
BLACK=`black --check python` | ||
BLACK_RETVAL=$? | ||
|
||
# Run flake8 and get results/return code | ||
FLAKE=`flake8 python` | ||
FLAKE_RETVAL=$? | ||
|
||
if [ "$BLACK_RETVAL" != "0" ]; then | ||
echo -e "\n\n>>>> FAILED: black style check; begin output\n\n" | ||
echo -e "$BLACK" | ||
echo -e "\n\n>>>> FAILED: black style check; end output\n\n" | ||
else | ||
echo -e "\n\n>>>> PASSED: black style check\n\n" | ||
fi | ||
|
||
if [ "$FLAKE_RETVAL" != "0" ]; then | ||
echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" | ||
echo -e "$FLAKE" | ||
echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" | ||
else | ||
echo -e "\n\n>>>> PASSED: flake8 style check\n\n" | ||
fi | ||
|
||
RETVALS=($BLACK_RETVAL $FLAKE_RETVAL) | ||
IFS=$'\n' | ||
RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` | ||
|
||
exit $RETVAL |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
#!/bin/bash | ||
# Copyright (c) 2018, NVIDIA CORPORATION. | ||
###################################### | ||
# cuDF CPU conda build script for CI # | ||
###################################### | ||
set -e | ||
|
||
# Logger function for build status output | ||
function logger() { | ||
echo -e "\n>>>> $@\n" | ||
} | ||
|
||
# Set path and build parallel level | ||
export PATH=/conda/bin:/usr/local/cuda/bin:$PATH | ||
export PARALLEL_LEVEL=4 | ||
|
||
# Set home to the job's workspace | ||
export HOME=$WORKSPACE | ||
|
||
# Switch to project root; also root of repo checkout | ||
cd $WORKSPACE | ||
|
||
# Get latest tag and number of commits since tag | ||
export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags` | ||
export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count` | ||
|
||
# If nightly build, append current YYMMDD to version | ||
if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then | ||
export VERSION_SUFFIX=`date +%y%m%d` | ||
fi | ||
|
||
################################################################################ | ||
# SETUP - Check environment | ||
################################################################################ | ||
|
||
logger "Get env..." | ||
env | ||
|
||
logger "Activate conda env..." | ||
source activate gdf | ||
|
||
logger "Check versions..." | ||
python --version | ||
gcc --version | ||
g++ --version | ||
conda list | ||
|
||
# FIX Added to deal with Anancoda SSL verification issues during conda builds | ||
conda config --set ssl_verify False | ||
|
||
################################################################################ | ||
# BUILD - Conda package builds (conda deps: libcudf <- libcudf_cffi <- cudf) | ||
################################################################################ | ||
|
||
logger "Build conda pkg for cuxfilter..." | ||
source ci/cpu/cuxfilter/build_cuXfilter.sh | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Lowercase script names. |
||
|
||
################################################################################ | ||
# UPLOAD - Conda packages | ||
################################################################################ | ||
|
||
logger "Upload conda pkgs..." | ||
source ci/cpu/upload_anaconda.sh |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
set -e | ||
|
||
echo "Building cuXfilter" | ||
conda build conda/recipes/cuXfilter --python=$PYTHON |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
#!/usr/bin/env bash | ||
|
||
#Upload cudf once per PYTHON | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. cudf --> cuxfilter |
||
if [[ "$CUDA" == "9.2" ]]; then | ||
export UPLOAD_CUDF=1 | ||
else | ||
export UPLOAD_CUDF=0 | ||
fi | ||
|
||
#Upload libcudf once per CUDA | ||
if [[ "$PYTHON" == "3.6" ]]; then | ||
export UPLOAD_LIBCUDF=1 | ||
else | ||
export UPLOAD_LIBCUDF=0 | ||
fi | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This block is unneeded as you are python only. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/bin/bash | ||
# | ||
# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh | ||
|
||
set -e | ||
|
||
export CUXFILTER_FILE=`conda build conda/recipes/cuXfilter --output` | ||
|
||
SOURCE_BRANCH=master | ||
CUDA_REL=${CUDA_VERSION%.*} | ||
|
||
# Restrict uploads to master branch | ||
if [ ${GIT_BRANCH} != ${SOURCE_BRANCH} ]; then | ||
echo "Skipping upload" | ||
return 0 | ||
fi | ||
|
||
if [ -z "$MY_UPLOAD_KEY" ]; then | ||
echo "No upload key" | ||
return 0 | ||
fi | ||
|
||
if [ "$UPLOAD_CUXFILTER" == "1" ]; then | ||
LABEL_OPTION="--label main --label cuda9.2 --label cuda10.0" | ||
echo "LABEL_OPTION=${LABEL_OPTION}" | ||
|
||
test -e ${CUXFILTER_FILE} | ||
echo "Upload cuXfilter" | ||
echo ${CUXFILTER_FILE} | ||
anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${CUXFILTER_FILE} | ||
|
||
fi |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
All lowercase args