Update GNUPlot to write outliers (#242)

* Update GNUPlot to write outliers * Skip brew update * Fix brew cache * Upgrade osx image * Remove cache fix * Disable warning * Try to speed up mac build * Break out dotnet * Switch mac job to anaconda * Add timing * Reduce reqs * Reduce build further * Fix bug
Illumina · Dec 4, 2020 · 23d1f27 · 23d1f27
1 parent 73c3c2a
commit 23d1f27
Show file tree

Hide file tree

Showing 8 changed files with 116 additions and 43 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -26,15 +26,31 @@ env:
 matrix:
   include:
   - os: osx
+    osx_image: xcode12.2
     compiler: clang
-    env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release ALL"
+    env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release Disable"
   - os: osx
+    osx_image: xcode12.2
+    compiler: clang
+    env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release DotNetStandard"
+  - os: osx
+    osx_image: xcode12.2
+    compiler: clang
+    env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 2.7.17"
+  - os: osx
+    osx_image: xcode12.2
+    compiler: clang
+    env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.5.9"
+  - os: osx
+    osx_image: xcode12.2
     compiler: clang
     env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.6.10"
   - os: osx
+    osx_image: xcode12.2
     compiler: clang
     env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.7.7"
   - os: osx
+    osx_image: xcode12.2
     compiler: clang
     env: DEPLOY_BUILD=true SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Release 3.8.2"
   - os: linux
@@ -49,6 +65,7 @@ matrix:
       - docker
     env: DEPLOY_BUILD=true DEPLOY_DOCS=true DOCKER_IMAGE=ezralanglois/interop SCRIPT="docker run --rm -v $PWD:/io ezralanglois/interop sh /io/tools/package.sh /io /io/dist travis OFF Release None"
   - os: osx
+    osx_image: xcode12.2
     compiler: clang
     env: SCRIPT="bash ./tools/package.sh $PWD $PWD/dist travis OFF Debug Disable"
   - os: linux

diff --git a/docs/src/changes.md b/docs/src/changes.md
@@ -1,11 +1,18 @@
 # Changes                                               {#changes}
 
+## v1.1.16
+
+Date       | Description
+---------- | -----------
+2020-11-30 | Issue-241: Write out outliers to GNUPlot file
+
+
 ## v1.1.15
 
 Date       | Description
 ---------- | -----------
-2020-08-26 | Issue-229: Support % loading concentration
-2020-08-26 | Issue-229: Fix Python binding for read_metrics_from_buffer
+2020-11-13 | Issue-229: Support % loading concentration
+2020-11-13 | Issue-229: Fix Python binding for read_metrics_from_buffer
 
 
 ## v1.1.14

diff --git a/interop/io/plot/gnuplot.h b/interop/io/plot/gnuplot.h
@@ -256,14 +256,29 @@ namespace illumina { namespace interop { namespace io { namespace plot
         {
             if (series.series_type() == model::plot::series<model::plot::candle_stick_point>::Candlestick)
             {
+                size_t max_num_outliers = 0;
                 for (size_t i = 0; i < series.size(); ++i)
                 {
-                    out << table::handle_nan(series[i].x()) << ",";
-                    out << table::handle_nan(series[i].lower()) << ",";
-                    out << table::handle_nan(series[i].p25()) << ",";
-                    out << table::handle_nan(series[i].p50()) << ",";
-                    out << table::handle_nan(series[i].p75()) << ",";
-                    out << table::handle_nan(series[i].upper());
+                    const model::plot::candle_stick_point point = series[i];
+                    const std::vector<float> outliers = point.outliers();
+                    max_num_outliers = std::max(max_num_outliers, outliers.size());
+                }
+
+                for (size_t i = 0; i < series.size(); ++i)
+                {
+                    const model::plot::candle_stick_point point = series[i];
+                    out << table::handle_nan(point.x()) << ",";
+                    out << table::handle_nan(point.lower()) << ",";
+                    out << table::handle_nan(point.p25()) << ",";
+                    out << table::handle_nan(point.p50()) << ",";
+                    out << table::handle_nan(point.p75()) << ",";
+                    out << table::handle_nan(point.upper());
+                    const std::vector<float> outliers = point.outliers();
+                    size_t j=0;
+                    for(;j<outliers.size();++j)
+                        out << "," << table::handle_nan(outliers[j]);
+                    for(;j<max_num_outliers;++j)
+                        out << "," << table::handle_nan(std::numeric_limits<float>::quiet_NaN());
                     out << std::endl;
                 }
             }

diff --git a/interop/logic/plot/plot_point.h b/interop/logic/plot/plot_point.h
@@ -44,7 +44,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot {
             util::outliers_lower(beg, end, lower, std::back_inserter(outliers));
             util::outliers_upper(beg, end, upper, std::back_inserter(outliers));
         }
-        size_t count = static_cast<size_t>(std::distance(beg,end));
+        const size_t count = static_cast<size_t>(std::distance(beg,end));
 
         I upper_it = std::lower_bound(beg, end, upper);// Not less
         I lower_it = std::lower_bound(beg, end, lower-(eps*lower));

diff --git a/src/ext/python/CMakeLists.txt b/src/ext/python/CMakeLists.txt
@@ -33,6 +33,12 @@ elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
         message(STATUS "Add flag -Wno-deprecated-declarations")
         set(PYTHON_GENERATED_COMPILE_FLAGS "${PYTHON_GENERATED_COMPILE_FLAGS} -Wno-deprecated-declarations")
     endif()
+    check_cxx_compiler_flag("-Wno-deprecated-register" IS_SUPPORTED_FLAG)
+    message(STATUS "Deprecated register disable warning: ${IS_SUPPORTED_FLAG}")
+    if(IS_SUPPORTED_FLAG)
+        message(STATUS "Add flag -Wno-deprecated-register")
+        set(PYTHON_GENERATED_COMPILE_FLAGS "${PYTHON_GENERATED_COMPILE_FLAGS} -Wno-deprecated-register")
+    endif()
 elseif(COMPILER_IS_GNUCC_OR_CLANG)
     set(PYTHON_GENERATED_COMPILE_FLAGS "${_WNO_UNINITIALIZED} ${_WNO_UNUSED_FUNCTION} ${_WNO_UNUSED_PARAMETER} ${_WNO_MAYBE_UNINITIALIZED} ${ENABLE_BIG_OBJ_FLAG} ${_WNO_STRICT_ALIASING} ${_FNO_STRICT_ALIASING}")
     if(WIN32 AND MINGW AND "${CMAKE_SIZEOF_VOID_P}" EQUAL "8")

diff --git a/src/interop/logic/plot/plot_by_cycle.cpp b/src/interop/logic/plot/plot_by_cycle.cpp
@@ -182,7 +182,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot
             size_t j=0;
             for(size_t cycle=0;cycle<m_max_cycle;++cycle)
             {
-                if(tile_by_cycle[cycle].empty())continue;
+                if(tile_by_cycle[cycle].empty())
+                    continue;
                 plot_candle_stick(m_points[j],
                                   tile_by_cycle[cycle].begin(),
                                   tile_by_cycle[cycle].end(),

diff --git a/tools/package.sh b/tools/package.sh
@@ -85,6 +85,18 @@ if [ ! -z "$8" ] ; then
     MORE_FLAGS="$8"
 fi
 
+echo "-------------------------------"
+echo "package.sh Configuration"
+echo "Source path: ${SOURCE_PATH}"
+echo "Artifact path: ${ARTIFACT_PATH}"
+echo "Build server: ${BUILD_SERVER}"
+echo "C89 Support: ${INTEROP_C89}"
+echo "Build Type: ${BUILD_TYPE}"
+echo "Python Version: ${PYTHON_VERSION}"
+echo "Build Number: ${ARTFACT_BUILD_NUMBER}"
+echo "Additional Flags: ${MORE_FLAGS}"
+echo "-------------------------------"
+
 CMAKE_EXTRA_FLAGS="-DDISABLE_PACKAGE_SUBDIR=${DISABLE_SUBDIR} -DENABLE_PORTABLE=ON -DENABLE_BACKWARDS_COMPATIBILITY=$INTEROP_C89 -DCMAKE_BUILD_TYPE=$BUILD_TYPE $MORE_FLAGS"
 
 
@@ -157,7 +169,7 @@ if [ -z $PYTHON_VERSION ] && [  -e /opt/python ] ; then
     done
 fi
 
-if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] ; then
+if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] && [ "$PYTHON_VERSION" != "DotNetStandard" ] ; then
     if [ "$PYTHON_VERSION" == "ALL" ] ; then
         # python_versions="2.7.17 3.5.9 3.6.10 3.7.7 3.8.2"
         python_versions="2.7.17 3.5.9"
@@ -170,9 +182,34 @@ if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] ; then
       CFLAGS="-I$(brew --prefix openssl)/include -I$(xcrun --show-sdk-path)/usr/include"
       LDFLAGS="-L$(brew --prefix openssl)/lib"
     fi
+    if [ -e "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
+      source $HOME/miniconda/etc/profile.d/conda.sh
+      conda config --set channel_priority strict
+      #conda update --all
+    fi
     for py_ver in $python_versions; do
         echo "Building Python $py_ver - $CFLAGS"
-        if hash pyenv 2> /dev/null; then
+        if [ -e "/Users/bioinformatics/anaconda3" ]; then
+          python_version=${py_ver}
+          conda remove --name py${python_version} --all -y || echo "py${python_version} not found"
+          echo "Create Python ${python_version}"
+          conda create --no-default-packages -n py${python_version} python=${python_version} -y # || conda create --no-default-packages -n py${python_version} python=${python_version} -y -c conda-forge
+
+          echo "Activate Python ${python_version}"
+          conda activate py${python_version}
+          conda env list
+          python -V
+          which python
+          echo "Install deps"
+          if [[ "$OSTYPE" == "darwin"* ]]; then
+            python -m pip install delocate
+          else
+            python -m pip install auditwheel==1.5
+          fi
+          conda install numpy -y --name py${python_version}
+          conda install wheel -y --name py${python_version}
+
+        elif hash pyenv 2> /dev/null; then
             export PATH=$(pyenv root)/shims:${PATH}
             if [[ "$OSTYPE" == "linux-gnu" ]]; then
                 if hash patchelf 2> /dev/null; then
@@ -221,20 +258,20 @@ if [ "$PYTHON_VERSION" != "" ] && [ "$PYTHON_VERSION" != "Disable" ] ; then
     done
 fi
 
-if [ ! -e $BUILD_PATH/CMakeCache.txt ] ; then
-    run "Configure" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS}
+
+if [ "$PYTHON_VERSION" == "Disable" ] ; then
+    run "Configure" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} -DENABLE_SWIG=OFF
     run "Build" cmake --build $BUILD_PATH -- -j${THREAD_COUNT}
     run "Test" cmake --build $BUILD_PATH --target check -- -j${THREAD_COUNT}
+    run "Package" cmake --build $BUILD_PATH --target bundle
 fi
 
-run "Package" cmake --build $BUILD_PATH --target bundle
-
+if [ "$PYTHON_VERSION" == "DotNetStandard" ] ; then
 
-if [ "$PYTHON_VERSION" != "Disable" ] ; then
   # Workaround for OSX
   export PATH=/usr/local/share/dotnet:${PATH}
   if hash dotnet 2> /dev/null; then
-      run "Configure DotNetStandard" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} -DCSBUILD_TOOL=DotNetStandard
+      run "Configure DotNetStandard" cmake $SOURCE_PATH -B${BUILD_PATH} ${CMAKE_EXTRA_FLAGS} -DCSBUILD_TOOL=DotNetStandard -DENABLE_PYTHON=OFF
       run "Test DotNetStandard" cmake --build $BUILD_PATH --target check -- -j${THREAD_COUNT}
       run "Package DotNetStandard" cmake --build $BUILD_PATH --target nupack -- -j${THREAD_COUNT}
   fi

diff --git a/tools/prereqs/travis-osx-install.sh b/tools/prereqs/travis-osx-install.sh
@@ -1,41 +1,31 @@
 #!/usr/bin/env bash
 
-sw_vers
+#sw_vers
 
-brew update > /dev/null
+#brew update > /dev/null
 #brew list
-which cmake || brew unlink cmake
-brew install cmake
-brew link cmake
+set +x
+brew list cmake > /dev/null || time brew upgrade cmake
 brew install zlib
-brew install swig@3
+time brew install swig@3
 brew unlink swig || true
 brew link swig@3 --force
-brew install doxygen
+time brew install doxygen
 brew install wget
-#brew install mono
-brew remove mono || true
 brew install nuget
 brew install coreutils || brew install gstat
-#brew install [email protected]
-brew uninstall openssl && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/8b9d6d688f483a0f33fcfc93d433de501b9c3513/Formula/openssl.rb
-brew outdated pyenv || brew upgrade pyenv
-brew cask install dotnet-sdk
-brew install readline xz
 
-echo "OpenSSL: $(brew --prefix openssl)"
+brew tap isen-ng/dotnet-sdk-versions
+brew cask list dotnet-sdk > /dev/null || time brew cask install dotnet-sdk2-2-400
+#brew cask list dotnet-sdk > /dev/null || time brew cask install dotnet-sdk
+
+time curl -o miniconda.sh  https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+time bash miniconda.sh -b -p $HOME/miniconda
+source $HOME/miniconda/etc/profile.d/conda.sh
 
-CFLAGS="-I$(brew --prefix openssl)/include -I$(xcrun --show-sdk-path)/usr/include"
-LDFLAGS="-L$(brew --prefix openssl)/lib"
-pyenv install 2.7.12
-pyenv global 2.7.12
+echo "OpenSSL: $(brew --prefix openssl)"
 export PATH=$(pyenv root)/shims:${PATH}
 
 which conda
-
-pip install numpy
-pip install wheel
-pip install delocate
-
 dotnet --version