From 23d1f270ecc91cde4acadceb878a758549487c50 Mon Sep 17 00:00:00 2001 From: Robert Langlois <ezralanglois@users.noreply.github.com> Date: Thu, 3 Dec 2020 19:22:01 -0800 Subject: [PATCH] Update GNUPlot to write outliers (#242) * Update GNUPlot to write outliers * Skip brew update * Fix brew cache * Upgrade osx image * Remove cache fix * Disable warning * Try to speed up mac build * Break out dotnet * Switch mac job to anaconda * Add timing * Reduce reqs * Reduce build further * Fix bug --- .travis.yml | 19 ++++++++- docs/src/changes.md | 11 ++++- interop/io/plot/gnuplot.h | 27 +++++++++--- interop/logic/plot/plot_point.h | 2 +- src/ext/python/CMakeLists.txt | 6 +++ src/interop/logic/plot/plot_by_cycle.cpp | 3 +- tools/package.sh | 53 ++++++++++++++++++++---- tools/prereqs/travis-osx-install.sh | 38 +++++++---------- 8 files changed, 116 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1cb4fd95d..fdeffbfe2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,15 +26,31 @@ env: matrix: include: - os: osx + osx_image: xcode12.2 compiler: clang - env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release ALL" + env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release Disable" - os: osx + osx_image: xcode12.2 + compiler: clang + env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release DotNetStandard" + - os: osx + osx_image: xcode12.2 + compiler: clang + env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 2.7.17" + - os: osx + osx_image: xcode12.2 + compiler: clang + env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.5.9" + - os: osx + osx_image: xcode12.2 compiler: clang env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.6.10" - os: osx + osx_image: xcode12.2 compiler: clang env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.7.7" - os: osx + osx_image: xcode12.2 compiler: clang env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.8.2" - os: linux @@ -49,6 +65,7 @@ matrix: - docker env: DEPLOY_BUILD=true DEPLOY_DOCS=true DOCKER_IMAGE=ezralanglois/interop SCRIPT="docker run --rm -v $PWD:/io ezralanglois/interop sh /io/tools/package.sh /io /io/dist travis OFF Release None" - os: osx + osx_image: xcode12.2 compiler: clang env: SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Debug Disable" - os: linux diff --git a/docs/src/changes.md b/docs/src/changes.md index 65964ea03..d42a3ecbb 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -1,11 +1,18 @@ # Changes {#changes} +## v1.1.16 + +Date | Description +---------- | ----------- +2020-11-30 | Issue-241: Write out outliers to GNUPlot file + + ## v1.1.15 Date | Description ---------- | ----------- -2020-08-26 | Issue-229: Support % loading concentration -2020-08-26 | Issue-229: Fix Python binding for read_metrics_from_buffer +2020-11-13 | Issue-229: Support % loading concentration +2020-11-13 | Issue-229: Fix Python binding for read_metrics_from_buffer ## v1.1.14 diff --git a/interop/io/plot/gnuplot.h b/interop/io/plot/gnuplot.h index 5e256c945..253db8d66 100644 --- a/interop/io/plot/gnuplot.h +++ b/interop/io/plot/gnuplot.h @@ -256,14 +256,29 @@ namespace illumina { namespace interop { namespace io { namespace plot { if (series.series_type() == model::plot::series<model::plot::candle_stick_point>::Candlestick) { + size_t max_num_outliers = 0; for (size_t i = 0; i < series.size(); ++i) { - out << table::handle_nan(series[i].x()) << ","; - out << table::handle_nan(series[i].lower()) << ","; - out << table::handle_nan(series[i].p25()) << ","; - out << table::handle_nan(series[i].p50()) << ","; - out << table::handle_nan(series[i].p75()) << ","; - out << table::handle_nan(series[i].upper()); + const model::plot::candle_stick_point point = series[i]; + const std::vector<float> outliers = point.outliers(); + max_num_outliers = std::max(max_num_outliers, outliers.size()); + } + + for (size_t i = 0; i < series.size(); ++i) + { + const model::plot::candle_stick_point point = series[i]; + out << table::handle_nan(point.x()) << ","; + out << table::handle_nan(point.lower()) << ","; + out << table::handle_nan(point.p25()) << ","; + out << table::handle_nan(point.p50()) << ","; + out << table::handle_nan(point.p75()) << ","; + out << table::handle_nan(point.upper()); + const std::vector<float> outliers = point.outliers(); + size_t j=0; + for(;j<outliers.size();++j) + out << "," << table::handle_nan(outliers[j]); + for(;j<max_num_outliers;++j) + out << "," << table::handle_nan(std::numeric_limits<float>::quiet_NaN()); out << std::endl; } } diff --git a/interop/logic/plot/plot_point.h b/interop/logic/plot/plot_point.h index 1ee29578c..e91458ee5 100644 --- a/interop/logic/plot/plot_point.h +++ b/interop/logic/plot/plot_point.h @@ -44,7 +44,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { util::outliers_lower(beg, end, lower, std::back_inserter(outliers)); util::outliers_upper(beg, end, upper, std::back_inserter(outliers)); } - size_t count = static_cast<size_t>(std::distance(beg,end)); + const size_t count = static_cast<size_t>(std::distance(beg,end)); I upper_it = std::lower_bound(beg, end, upper);// Not less I lower_it = std::lower_bound(beg, end, lower-(eps*lower)); diff --git a/src/ext/python/CMakeLists.txt b/src/ext/python/CMakeLists.txt index a4c3385d5..1c700d7e6 100644 --- a/src/ext/python/CMakeLists.txt +++ b/src/ext/python/CMakeLists.txt @@ -33,6 +33,12 @@ elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") message(STATUS "Add flag -Wno-deprecated-declarations") set(PYTHON_GENERATED_COMPILE_FLAGS "${PYTHON_GENERATED_COMPILE_FLAGS} -Wno-deprecated-declarations") endif() + check_cxx_compiler_flag("-Wno-deprecated-register" IS_SUPPORTED_FLAG) + message(STATUS "Deprecated register disable warning: ${IS_SUPPORTED_FLAG}") + if(IS_SUPPORTED_FLAG) + message(STATUS "Add flag -Wno-deprecated-register") + set(PYTHON_GENERATED_COMPILE_FLAGS "${PYTHON_GENERATED_COMPILE_FLAGS} -Wno-deprecated-register") + endif() elseif(COMPILER_IS_GNUCC_OR_CLANG) set(PYTHON_GENERATED_COMPILE_FLAGS "${_WNO_UNINITIALIZED} ${_WNO_UNUSED_FUNCTION} ${_WNO_UNUSED_PARAMETER} ${_WNO_MAYBE_UNINITIALIZED} ${ENABLE_BIG_OBJ_FLAG} ${_WNO_STRICT_ALIASING} ${_FNO_STRICT_ALIASING}") if(WIN32 AND MINGW AND "${CMAKE_SIZEOF_VOID_P}" EQUAL "8") diff --git a/src/interop/logic/plot/plot_by_cycle.cpp b/src/interop/logic/plot/plot_by_cycle.cpp index a326f41ef..55c77f7dc 100644 --- a/src/interop/logic/plot/plot_by_cycle.cpp +++ b/src/interop/logic/plot/plot_by_cycle.cpp @@ -182,7 +182,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot size_t j=0; for(size_t cycle=0;cycle<m_max_cycle;++cycle) { - if(tile_by_cycle[cycle].empty())continue; + if(tile_by_cycle[cycle].empty()) + continue; plot_candle_stick(m_points[j], tile_by_cycle[cycle].begin(), tile_by_cycle[cycle].end(), diff --git a/tools/package.sh b/tools/package.sh index f3da5e4a8..da1a4a219 100644 --- a/tools/package.sh +++ b/tools/package.sh @@ -85,6 +85,18 @@ if [ ! -z "$8" ] ; then MORE_FLAGS="$8" fi +echo "-------------------------------" +echo "package.sh Configuration" +echo "Source path: ${SOURCE_PATH}" +echo "Artifact path: ${ARTIFACT_PATH}" +echo "Build server: ${BUILD_SERVER}" +echo "C89 Support: ${INTEROP_C89}" +echo "Build Type: ${BUILD_TYPE}" +echo "Python Version: ${PYTHON_VERSION}" +echo "Build Number: ${ARTFACT_BUILD_NUMBER}" +echo "Additional Flags: ${MORE_FLAGS}" +echo "-------------------------------" + CMAKE_EXTRA_FLAGS="-DDISABLE_PACKAGE_SUBDIR=${DISABLE_SUBDIR} -DENABLE_PORTABLE=ON -DENABLE_BACKWARDS_COMPATIBILITY=$INTEROP_C89 -DCMAKE_BUILD_TYPE=$BUILD_TYPE $MORE_FLAGS" @@ -157,7 +169,7 @@ if [ -z $PYTHON_VERSION ] && [ -e /opt/python ] ; then done fi -if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] ; then +if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] && [ "$PYTHON_VERSION" != "DotNetStandard" ] ; then if [ "$PYTHON_VERSION" == "ALL" ] ; then # python_versions="2.7.17 3.5.9 3.6.10 3.7.7 3.8.2" python_versions="2.7.17 3.5.9" @@ -170,9 +182,34 @@ if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] ; then CFLAGS="-I$(brew --prefix openssl)/include -I$(xcrun --show-sdk-path)/usr/include" LDFLAGS="-L$(brew --prefix openssl)/lib" fi + if [ -e "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + source $HOME/miniconda/etc/profile.d/conda.sh + conda config --set channel_priority strict + #conda update --all + fi for py_ver in $python_versions; do echo "Building Python $py_ver - $CFLAGS" - if hash pyenv 2> /dev/null; then + if [ -e "/Users/bioinformatics/anaconda3" ]; then + python_version=${py_ver} + conda remove --name py${python_version} --all -y || echo "py${python_version} not found" + echo "Create Python ${python_version}" + conda create --no-default-packages -n py${python_version} python=${python_version} -y # || conda create --no-default-packages -n py${python_version} python=${python_version} -y -c conda-forge + + echo "Activate Python ${python_version}" + conda activate py${python_version} + conda env list + python -V + which python + echo "Install deps" + if [[ "$OSTYPE" == "darwin"* ]]; then + python -m pip install delocate + else + python -m pip install auditwheel==1.5 + fi + conda install numpy -y --name py${python_version} + conda install wheel -y --name py${python_version} + + elif hash pyenv 2> /dev/null; then export PATH=$(pyenv root)/shims:${PATH} if [[ "$OSTYPE" == "linux-gnu" ]]; then if hash patchelf 2> /dev/null; then @@ -221,20 +258,20 @@ if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] ; then done fi -if [ ! -e $BUILD_PATH/CMakeCache.txt ] ; then - run "Configure" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} + +if [ "$PYTHON_VERSION" == "Disable" ] ; then + run "Configure" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} -DENABLE_SWIG=OFF run "Build" cmake --build $BUILD_PATH -- -j${THREAD_COUNT} run "Test" cmake --build $BUILD_PATH --target check -- -j${THREAD_COUNT} + run "Package" cmake --build $BUILD_PATH --target bundle fi -run "Package" cmake --build $BUILD_PATH --target bundle - +if [ "$PYTHON_VERSION" == "DotNetStandard" ] ; then -if [ "$PYTHON_VERSION" != "Disable" ] ; then # Workaround for OSX export PATH=/usr/local/share/dotnet:${PATH} if hash dotnet 2> /dev/null; then - run "Configure DotNetStandard" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} -DCSBUILD_TOOL=DotNetStandard + run "Configure DotNetStandard" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} -DCSBUILD_TOOL=DotNetStandard -DENABLE_PYTHON=OFF run "Test DotNetStandard" cmake --build $BUILD_PATH --target check -- -j${THREAD_COUNT} run "Package DotNetStandard" cmake --build $BUILD_PATH --target nupack -- -j${THREAD_COUNT} fi diff --git a/tools/prereqs/travis-osx-install.sh b/tools/prereqs/travis-osx-install.sh index 8d9aa9bd6..37e307bab 100644 --- a/tools/prereqs/travis-osx-install.sh +++ b/tools/prereqs/travis-osx-install.sh @@ -1,41 +1,31 @@ #!/usr/bin/env bash -sw_vers +#sw_vers -brew update > /dev/null +#brew update > /dev/null #brew list -which cmake || brew unlink cmake -brew install cmake -brew link cmake +set +x +brew list cmake > /dev/null || time brew upgrade cmake brew install zlib -brew install swig@3 +time brew install swig@3 brew unlink swig || true brew link swig@3 --force -brew install doxygen +time brew install doxygen brew install wget -#brew install mono -brew remove mono || true brew install nuget brew install coreutils || brew install gstat -#brew install openssl@1.1 -brew uninstall openssl && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/8b9d6d688f483a0f33fcfc93d433de501b9c3513/Formula/openssl.rb -brew outdated pyenv || brew upgrade pyenv -brew cask install dotnet-sdk -brew install readline xz -echo "OpenSSL: $(brew --prefix openssl)" +brew tap isen-ng/dotnet-sdk-versions +brew cask list dotnet-sdk > /dev/null || time brew cask install dotnet-sdk2-2-400 +#brew cask list dotnet-sdk > /dev/null || time brew cask install dotnet-sdk + +time curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +time bash miniconda.sh -b -p $HOME/miniconda +source $HOME/miniconda/etc/profile.d/conda.sh -CFLAGS="-I$(brew --prefix openssl)/include -I$(xcrun --show-sdk-path)/usr/include" -LDFLAGS="-L$(brew --prefix openssl)/lib" -pyenv install 2.7.12 -pyenv global 2.7.12 +echo "OpenSSL: $(brew --prefix openssl)" export PATH=$(pyenv root)/shims:${PATH} which conda - -pip install numpy -pip install wheel -pip install delocate - dotnet --version