Merge pull request #74 from raydouglass/gpuci-conda

[REVIEW] Add CI scripts and conda recipe

ci/checks/changelog.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+###########################
+# gQuant CHANGELOG Tester #
+###########################
+
+# Checkout master for comparison
+git checkout --quiet master
+
+# Switch back to tip of PR branch
+git checkout --quiet current-pr-branch
+
+# Ignore errors during searching
+set +e
+
+# Get list of modified files between matster and PR branch
+CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md`
+# Check if CHANGELOG has PR ID
+PRNUM=`cat CHANGELOG.md | grep "$PR_ID"`
+RETVAL=0
+
+# Return status of check result
+if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then
+  echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n"
+else
+  echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n"
+  RETVAL=1
+fi
+
+cat << EOF
+  It should be placed under the section for the appropriate release.
+  It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate.
+  It should be formatted as '- PR #<PR number> <Concise human-readable description of the PR's new feature, improvement, or bug fix>'
+    Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs'
+
+
+EOF
+
+exit $RETVAL

ci/checks/style.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.
+################################################################################
+# gQuant Style Tester
+################################################################################
+
+# Ignore errors and set path
+set +e
+PATH=/conda/bin:$PATH
+
+# Activate common conda env
+source activate gdf
+
+# Run flake8 and get results/return code
+FLAKE=`flake8 python`
+RETVAL=$?
+
+# Output results if failure otherwise show pass
+if [ "$FLAKE" != "" ]; then
+  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
+  echo -e "$FLAKE"
+  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
+fi
+
+exit $RETVAL

ci/common/prebuild.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+export RAPIDS_VERSION="0.10"

ci/cpu/build.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Copyright (c) 2019, NVIDIA CORPORATION.
+################################################################################
+# gQuant cpu build
+################################################################################
+set -e
+
+# Logger function for build status output
+function logger() {
+  echo -e "\n>>>> $@\n"
+}
+
+# Set path and build parallel level
+export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# Get latest tag and number of commits since tag
+export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags`
+export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count`
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+logger "Get env..."
+env
+
+logger "Activate conda env..."
+source activate gdf
+
+logger "Check versions..."
+python --version
+gcc --version
+g++ --version
+conda list
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+# If nightly build, append current YYMMDD to version
+if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
+  export VERSION_SUFFIX=`date +%y%m%d`
+fi
+
+################################################################################
+# BUILD - Conda package build
+################################################################################
+
+conda build conda/recipes/gquant --python=${PYTHON}
+
+################################################################################
+# UPLOAD - Conda package
+################################################################################
+
+logger "Upload conda pkg..."
+source ci/cpu/upload-anaconda.sh

ci/cpu/prebuild.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+source $WORKSPACE/ci/common/prebuild.sh

ci/cpu/upload-anaconda.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+export UPLOADFILE=`conda build conda/recipes/gquant --python=$PYTHON --output`
+CUDA_REL=${CUDA_VERSION%.*}
+
+SOURCE_BRANCH=master
+
+LABEL_OPTION="--label main"
+echo "LABEL_OPTION=${LABEL_OPTION}"
+
+# Restrict uploads to master branch
+if [ ${GIT_BRANCH} != ${SOURCE_BRANCH} ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+if [ -z "$MY_UPLOAD_KEY" ]; then
+    echo "No upload key"
+    return 0
+fi
+
+echo "Upload"
+echo ${UPLOADFILE}
+anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${UPLOADFILE}

ci/gpu/build.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+###########################################
+# gQuant GPU build and test script for CI #
+###########################################
+set -e
+NUMARGS=$#
+ARGS=$*
+
+# Logger function for build status output
+function logger() {
+  echo -e "\n>>>> $@\n"
+}
+
+# Arg parsing function
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+# Set path and build parallel level
+export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=4
+export CUDA_REL=${CUDA_VERSION%.*}
+export CUDA_REL2=${CUDA//./}
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Parse git describe
+cd $WORKSPACE
+export GIT_DESCRIBE_TAG=`git describe --tags`
+export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+
+# Enable NumPy's __array_function__ protocol (needed for NumPy 1.16.x,
+# will possibly be enabled by default starting on 1.17)
+export NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+logger "Check environment..."
+env
+
+logger "Check GPU usage..."
+nvidia-smi
+
+logger "Activate conda env..."
+source activate gdf
+conda list
+
+logger "Install dependencies"
+conda install -y "cudf=${RAPIDS_VERSION:-0.10}" "dask-cudf=${RAPIDS_VERSION:-0.10}" networkx "bqplot=0.11.5" xgboost
+
+logger "Check versions..."
+python --version
+$CC --version
+$CXX --version
+conda list
+
+################################################################################
+# BUILD - Build gQuant
+################################################################################
+
+logger "Build gQuant..."
+cd $WORKSPACE
+python -m pip install -e .
+
+
+################################################################################
+# TEST - Run py.tests for gQuant
+################################################################################
+
+if hasArg --skip-tests; then
+    logger "Skipping Tests..."
+else
+    logger "Python py.test for gQuant..."
+    cd $WORKSPACE
+    py.test -vs --cache-clear --junitxml=${WORKSPACE}/junit-gquant.xml --cov-config=.coveragerc --cov=gquant --cov-report=xml:${WORKSPACE}/gquant-coverage.xml --cov-report term tests/
+fi

ci/gpu/prebuild.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+source $WORKSPACE/ci/common/prebuild.sh

ci/release/update-version.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.
+################################################################################
+# dask-cuda version updater
+################################################################################
+
+## Usage
+# bash update-version.sh <type>
+#     where <type> is either `major`, `minor`, `patch`
+
+set -e
+
+# Grab argument for release type
+RELEASE_TYPE=$1
+
+# Get current version and calculate next versions
+CURRENT_TAG=`git tag | grep -xE 'v[0-9\.]+' | sort --version-sort | tail -n 1 | tr -d 'v'`
+CURRENT_MAJOR=`echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}'`
+CURRENT_MINOR=`echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}'`
+CURRENT_PATCH=`echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}'`
+NEXT_MAJOR=$((CURRENT_MAJOR + 1))
+NEXT_MINOR=$((CURRENT_MINOR + 1))
+NEXT_PATCH=$((CURRENT_PATCH + 1))
+NEXT_FULL_TAG=""
+NEXT_SHORT_TAG=""
+
+# Determine release type
+if [ "$RELEASE_TYPE" == "major" ]; then
+  NEXT_FULL_TAG="${NEXT_MAJOR}.0.0"
+  NEXT_SHORT_TAG="${NEXT_MAJOR}.0"
+elif [ "$RELEASE_TYPE" == "minor" ]; then
+  NEXT_FULL_TAG="${CURRENT_MAJOR}.${NEXT_MINOR}.0"
+  NEXT_SHORT_TAG="${CURRENT_MAJOR}.${NEXT_MINOR}"
+elif [ "$RELEASE_TYPE" == "patch" ]; then
+  NEXT_FULL_TAG="${CURRENT_MAJOR}.${CURRENT_MINOR}.${NEXT_PATCH}"
+  NEXT_SHORT_TAG="${CURRENT_MAJOR}.${CURRENT_MINOR}"
+else
+  echo "Incorrect release type; use 'major', 'minor', or 'patch' as an argument"
+  exit 1
+fi
+
+echo "Preparing '$RELEASE_TYPE' release [$CURRENT_TAG -> $NEXT_FULL_TAG]"
+
+# Inplace sed replace; workaround for Linux and Mac
+function sed_runner() {
+    sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak
+}
+
+
+# No-op

conda/recipes/gquant/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+python setup.py install --single-version-externally-managed --record=record.txt

conda/recipes/gquant/meta.yaml

@@ -0,0 +1,46 @@
+# Copyright (c) 2019, NVIDIA CORPORATION.
+
+# Usage:
+#   conda build -c conda-forge -c defaults .
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+{% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
+{% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
+{% set py_version=environ.get('CONDA_PY', 36) %}
+{% set py_version=environ.get('RAPIDS_VERSION', '0.10') %}
+package:
+  name: gquant
+  version: {{ version }}
+
+source:
+  path: ../../..
+
+build:
+  number: {{ git_revision_count }}
+  string: py{{ py_version }}_{{ git_revision_count }}
+  script_env:
+    - VERSION_SUFFIX
+    - RAPIDS_VERSION
+
+requirements:
+  build:
+    - python x.x
+    - setuptools
+  run:
+    - python x.x
+    - cudf ={{ rapids_version }}
+    - dask-cudf ={{ rapids_version }}
+    - networkx
+    - bqplot=0.11.5
+    - xgboost
+
+test:
+  imports:
+    - gquant
+    - gquant.dataframe_flow
+
+
+about:
+  home: http://rapids.ai/
+  license: Apache-2.0
+  license_file: ../../../LICENSE
+  summary: gquant library