From d1a0a802f437fc4baf1ade663e1ec900ee3f4cf8 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Mon, 5 Dec 2016 09:25:06 -0800 Subject: [PATCH 1/4] IPA-5734: Synchronize master --- CMakeLists.txt | 4 +- README.md | 1 + cmake/AssemblyInfo.cs.in | 9 + cmake/InternalUtils.cmake | 17 +- cmake/Modules/CheckIsNaN.cmake | 4 +- cmake/Modules/FindCSharp.cmake | 10 +- cmake/Modules/FindGMock.cmake | 5 +- cmake/Modules/FindGTest.cmake | 6 +- cmake/package.nuspec.in | 4 +- cmake/package.targets | 4 +- cmake/version.rc.in | 50 + docs/src/changes.md | 16 + docs/src/contribute.md | 9 + docs/src/install.md | 2 +- interop/constants/enum_description.h | 75 + interop/constants/enums.h | 92 +- interop/constants/typedefs.h | 3 +- interop/external/rapidxml.hpp | 8 +- interop/io/format/abstract_metric_format.h | 12 + interop/io/format/default_layout.h | 24 + interop/io/format/map_io.h | 47 +- interop/io/format/metric_format.h | 43 +- interop/io/format/metric_format_factory.h | 4 +- interop/io/format/stream_util.h | 5 +- interop/io/layout/base_metric.h | 72 +- interop/io/metric_file_stream.h | 83 +- interop/io/metric_stream.h | 185 ++- interop/io/paths.h | 166 +++ interop/io/plot/gnuplot.h | 37 +- interop/io/table/csv_format.h | 105 +- interop/logic/metric/metric_value.h | 15 +- interop/logic/metric/q_metric.h | 10 +- interop/logic/metric/tile_metric.h | 2 +- interop/logic/plot/plot_flowcell_map.h | 56 + interop/logic/plot/plot_qscore_heatmap.h | 3 +- interop/logic/plot/plot_qscore_histogram.h | 1 + interop/logic/summary/cycle_state_summary.h | 13 +- interop/logic/summary/error_summary.h | 7 +- interop/logic/summary/quality_summary.h | 2 +- interop/logic/summary/run_summary.h | 4 +- interop/logic/summary/tile_summary.h | 54 +- .../logic/table/check_imaging_table_column.h | 27 +- interop/logic/table/create_imaging_table.h | 11 +- .../table/create_imaging_table_columns.h | 15 +- interop/logic/table/table_populator.h | 22 +- interop/logic/utils/channel.h | 4 +- interop/logic/utils/enums.h | 40 + interop/logic/utils/metric_type_ext.h | 2 + interop/logic/utils/metrics_to_load.h | 1 + interop/model/metric_base/base_cycle_metric.h | 36 +- interop/model/metric_base/base_metric.h | 27 +- interop/model/metric_base/base_read_metric.h | 23 +- interop/model/metric_base/metric_set.h | 212 ++- .../metrics/corrected_intensity_metric.h | 16 +- interop/model/metrics/extraction_metric.h | 11 +- interop/model/metrics/index_metric.h | 14 +- interop/model/metrics/q_by_lane_metric.h | 5 +- interop/model/metrics/q_metric.h | 5 +- interop/model/metrics/tile_metric.h | 58 +- interop/model/plot/axes.h | 48 + interop/model/plot/bar_point.h | 17 + interop/model/plot/candle_stick_point.h | 28 + interop/model/plot/chart_data.h | 20 + interop/model/plot/data_point.h | 19 + interop/model/plot/data_point_collection.h | 1 + interop/model/plot/filter_options.h | 296 +++- interop/model/plot/flowcell_data.h | 59 + interop/model/plot/heatmap_data.h | 34 + interop/model/plot/plot_data.h | 28 + interop/model/plot/series.h | 44 +- interop/model/run/flowcell_layout.h | 24 + interop/model/run/info.h | 20 +- interop/model/run/parameters.h | 2 +- interop/model/run_metrics.h | 229 +-- interop/model/summary/index_count_summary.h | 23 +- .../model/summary/index_flowcell_summary.h | 14 + interop/model/summary/index_lane_summary.h | 26 +- interop/model/summary/lane_summary.h | 33 +- interop/model/summary/metric_summary.h | 27 +- interop/model/summary/read_summary.h | 5 +- interop/model/summary/run_summary.h | 80 +- interop/model/summary/stat_summary.h | 20 +- interop/model/summary/surface_summary.h | 4 +- interop/util/constant_mapping.h | 48 +- interop/util/cstdint.h | 2 + interop/util/filesystem.h | 11 + interop/util/indirect_range_iterator.h | 228 +++ interop/util/length_of.h | 12 + interop/util/lexical_cast.h | 57 +- interop/util/map.h | 23 + interop/util/math.h | 2 +- interop/util/timer.h | 26 + interop/util/type_traits.h | 4 + src/apps/CMakeLists.txt | 11 +- src/apps/cyclesim.cpp | 17 +- src/apps/dumpbin.cpp | 16 +- src/apps/imaging_table.cpp | 2 + src/apps/inc/application.h | 11 +- src/apps/interop2csv.cpp | 34 +- src/apps/summary.cpp | 43 +- src/examples/csharp/Example3.cs | 2 +- src/examples/example1.cpp | 2 +- src/examples/example2.cpp | 2 +- src/examples/example3.cpp | 21 +- src/examples/example4.cpp | 12 +- src/ext/CMakeLists.txt | 13 +- src/ext/csharp/CMakeLists.txt | 44 +- src/ext/swig/arrays/arrays_impl.i | 1 - src/ext/swig/arrays/arrays_numpy_impl.i | 7 +- src/ext/swig/arrays/numpy.i | 16 +- src/ext/swig/comm.i | 8 + src/ext/swig/exceptions/exceptions_impl.i | 9 + src/ext/swig/extends/extends_csharp.i | 242 ---- src/ext/swig/extends/extends_impl.i | 13 - src/ext/swig/metrics.i | 49 +- src/ext/swig/plot.i | 9 + src/ext/swig/run.i | 41 + src/ext/swig/summary.i | 11 +- src/ext/swig/{imaging.i => table.i} | 12 +- src/interop/CMakeLists.txt | 25 +- .../logic/metric/extraction_metric.cpp | 6 +- src/interop/logic/metric/q_metric.cpp | 8 +- src/interop/logic/plot/plot_by_cycle.cpp | 25 +- src/interop/logic/plot/plot_by_lane.cpp | 8 +- src/interop/logic/plot/plot_flowcell_map.cpp | 18 +- .../logic/plot/plot_qscore_heatmap.cpp | 32 +- .../logic/plot/plot_qscore_histogram.cpp | 55 +- src/interop/logic/plot/plot_sample_qc.cpp | 23 +- src/interop/logic/summary/index_summary.cpp | 30 +- src/interop/logic/summary/run_summary.cpp | 94 +- .../logic/table/create_imaging_table.cpp | 71 +- .../table/create_imaging_table_columns.cpp | 44 +- .../metrics/corrected_intensity_metric.cpp | 400 +++--- src/interop/model/metrics/error_metric.cpp | 179 +-- .../model/metrics/extraction_metric.cpp | 264 ++-- src/interop/model/metrics/image_metric.cpp | 475 ++++--- src/interop/model/metrics/index_metric.cpp | 17 +- .../model/metrics/q_collapsed_metric.cpp | 29 +- src/interop/model/metrics/q_metric.cpp | 1242 ++++++++++------- src/interop/model/metrics/tile_metric.cpp | 451 +++--- src/interop/model/run/info.cpp | 4 +- src/interop/model/run/parameters.cpp | 8 +- src/interop/model/run_metrics.cpp | 133 +- src/interop/model/summary/index_summary.cpp | 152 ++ src/interop/model/summary/run_summary.cpp | 92 +- src/interop/util/filesystem.cpp | 14 + src/interop/util/time.cpp | 6 +- src/tests/csharp/CMakeLists.txt | 5 +- src/tests/csharp/logic/ExceptionTest.cs | 5 +- src/tests/csharp/logic/ImagingTableLogic.cs | 16 +- src/tests/csharp/logic/PlotDataByCycleTest.cs | 2 +- .../metrics/CorrectedIntensityMetricsTest.cs | 31 +- src/tests/csharp/metrics/ErrorMetricsTest.cs | 18 +- .../csharp/metrics/ExtractionMetricsTest.cs | 18 +- src/tests/csharp/metrics/ImageMetricsTest.cs | 18 +- src/tests/csharp/metrics/IndexMetricsTest.cs | 22 +- src/tests/csharp/metrics/PerformanceTest.cs | 6 +- src/tests/csharp/metrics/QMetricsTest.cs | 44 +- src/tests/csharp/metrics/RunMetricsTest.cs | 52 + src/tests/csharp/metrics/TileMetricsTest.cs | 28 +- src/tests/csharp/run/RunInfoTest.cs | 138 ++ src/tests/csharp/run/RunParametersTest.cs | 150 ++ src/tests/interop/CMakeLists.txt | 28 +- .../inc/abstract_regression_test_generator.h | 63 +- src/tests/interop/inc/generic_fixture.h | 226 ++- .../interop/inc/proxy_parameter_generator.h | 51 +- src/tests/interop/inc/regression_test_data.h | 11 +- src/tests/interop/logic/channel_test.cpp | 379 +++++ src/tests/interop/logic/enum_parsing_test.cpp | 25 +- .../logic/imaging_table_logic_test.cpp | 29 +- .../logic/imaging_table_regression_test.cpp | 196 ++- .../logic/inc/empty_plot_test_generator.h | 99 ++ .../logic/inc/metric_filter_iterator.h | 266 ++++ .../inc/plot_regression_test_generator.h | 177 +++ .../interop/logic/index_summary_test.cpp | 364 +++++ src/tests/interop/logic/plot_bar_test.cpp | 154 ++ .../interop/logic/plot_candle_stick_test.cpp | 182 +++ .../interop/logic/plot_flowcell_test.cpp | 134 ++ src/tests/interop/logic/plot_heatmap_test.cpp | 126 ++ src/tests/interop/logic/plot_logic_test.cpp | 479 +++++-- .../interop/logic/summary_metrics_test.cpp | 787 +++++++---- .../interop/metrics/base_metric_tests.cpp | 37 +- .../corrected_intensity_metrics_test.cpp | 8 +- src/tests/interop/metrics/coverage_test.cpp | 105 ++ .../interop/metrics/error_metrics_test.cpp | 7 +- .../metrics/extraction_metrics_test.cpp | 5 +- .../interop/metrics/image_metrics_test.cpp | 7 +- .../inc/corrected_intensity_metrics_test.h | 24 +- .../interop/metrics/inc/error_metrics_test.h | 5 +- .../metrics/inc/extraction_metrics_test.h | 5 +- .../interop/metrics/inc/format_registry.h | 94 ++ .../interop/metrics/inc/image_metrics_test.h | 4 +- .../interop/metrics/inc/index_metrics_test.h | 3 +- .../metrics/inc/metric_format_fixtures.h | 35 +- .../interop/metrics/inc/metric_generator.h | 103 +- .../metrics/inc/q_collapsed_metrics_test.h | 73 +- .../interop/metrics/inc/q_metrics_test.h | 20 +- .../interop/metrics/inc/tile_metrics_test.h | 5 +- .../interop/metrics/index_metrics_test.cpp | 9 +- .../metrics/metric_stream_error_test.cpp | 137 ++ .../interop/metrics/metric_streams_test.cpp | 90 +- .../interop/metrics/q_by_lane_metric_test.cpp | 4 +- .../metrics/q_collapsed_metrics_test.cpp | 15 +- src/tests/interop/metrics/q_metrics_test.cpp | 9 +- src/tests/interop/metrics/run_metric_test.cpp | 5 +- .../interop/metrics/tile_metrics_test.cpp | 73 +- src/tests/interop/unit_tests.cpp | 15 +- src/tests/interop/util/stat_test.cpp | 13 +- tools/build_test.bat | 5 - tools/hooks/pre-commit.sh | 7 + tools/package.bat | 57 + tools/package.sh | 71 + tools/regression_test.ps1 | 91 ++ 213 files changed, 9666 insertions(+), 3669 deletions(-) create mode 100644 cmake/AssemblyInfo.cs.in create mode 100644 cmake/version.rc.in create mode 100644 interop/constants/enum_description.h create mode 100644 interop/io/paths.h create mode 100644 interop/util/indirect_range_iterator.h create mode 100644 interop/util/map.h create mode 100644 interop/util/timer.h delete mode 100644 src/ext/swig/extends/extends_csharp.i delete mode 100644 src/ext/swig/extends/extends_impl.i rename src/ext/swig/{imaging.i => table.i} (90%) create mode 100644 src/interop/model/summary/index_summary.cpp create mode 100644 src/tests/csharp/metrics/RunMetricsTest.cs create mode 100644 src/tests/csharp/run/RunInfoTest.cs create mode 100644 src/tests/csharp/run/RunParametersTest.cs create mode 100644 src/tests/interop/logic/channel_test.cpp create mode 100644 src/tests/interop/logic/inc/empty_plot_test_generator.h create mode 100644 src/tests/interop/logic/inc/metric_filter_iterator.h create mode 100644 src/tests/interop/logic/inc/plot_regression_test_generator.h create mode 100644 src/tests/interop/logic/index_summary_test.cpp create mode 100644 src/tests/interop/logic/plot_bar_test.cpp create mode 100644 src/tests/interop/logic/plot_candle_stick_test.cpp create mode 100644 src/tests/interop/logic/plot_flowcell_test.cpp create mode 100644 src/tests/interop/logic/plot_heatmap_test.cpp create mode 100644 src/tests/interop/metrics/coverage_test.cpp create mode 100644 src/tests/interop/metrics/inc/format_registry.h create mode 100644 src/tests/interop/metrics/metric_stream_error_test.cpp create mode 100644 tools/package.bat create mode 100644 tools/package.sh create mode 100644 tools/regression_test.ps1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 88e1ecc20..3160180f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ include(${PROJECT_SOURCE_DIR}/cmake/Modules/UseGitVersion.cmake) include(${PROJECT_SOURCE_DIR}/cmake/InternalUtils.cmake) set(ARCHIVE_VERSION "v1.0.13-src") + if (NOT CMAKE_BUILD_TYPE) message(STATUS "No build type selected, default to Release") set(CMAKE_BUILD_TYPE "Release") @@ -34,6 +35,7 @@ option(ENABLE_CSHARP "Build C# language bindings" ON) # Options to control integration builds set(PACKAGE_SUFFIX "" CACHE STRING "Suffix added to the output packages") set(BUILD_NUMBER "" CACHE STRING "Build number used for select packing scripts") +mark_as_advanced(PACKAGE_SUFFIX BUILD_NUMBER) include_directories(.) add_version_target(version ${CMAKE_SOURCE_DIR}/interop/version.h INTEROP_VERSION ${ARCHIVE_VERSION}) @@ -89,7 +91,7 @@ find_package(Git) if(GIT_FOUND) string(REGEX REPLACE "[^v]*(v[0-9]+.[0-9]+.[0-9]+)-.*" "\\1" TAG ${INTEROP_VERSION}) add_custom_target(history - COMMAND ${GIT_EXECUTABLE};--no-pager;log;${TAG}..HEAD;--date=short;--format='%ad | %B' + COMMAND ${GIT_EXECUTABLE};log;${TAG}..HEAD;-m;--first-parent;--date=short;--format='%ad | %b' COMMENT "List all commits from last tag - ${TAG}" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) endif() diff --git a/README.md b/README.md index 0c9d08f76..2292d934d 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ There are several known limitations to the current library: 4. We do not support Mono on Windows 5. If both Visual Studio and Mono are installed, the build script will only use Visual Studio for C# 6. We do not support 32-bit builds + 7. MinGW W64 4.9.x and prior will not link properly SAV Analysis Tab ---------------- diff --git a/cmake/AssemblyInfo.cs.in b/cmake/AssemblyInfo.cs.in new file mode 100644 index 000000000..b9048db86 --- /dev/null +++ b/cmake/AssemblyInfo.cs.in @@ -0,0 +1,9 @@ +using System.Reflection; + +[assembly: AssemblyTitle("Illumina.InterOp")] +[assembly: AssemblyCompany("Illumina")] +[assembly: AssemblyProduct("Illumina.InterOp")] +[assembly: AssemblyCopyright("Copyright © Illumina 2015")] +[assembly: AssemblyVersion("@VERSION_SHORT@")] +[assembly: AssemblyFileVersion("@VERSION_SHORT@")] + diff --git a/cmake/InternalUtils.cmake b/cmake/InternalUtils.cmake index 63c78c8a2..2d3c86a68 100644 --- a/cmake/InternalUtils.cmake +++ b/cmake/InternalUtils.cmake @@ -66,6 +66,11 @@ macro(interop_config_compiler_and_linker) check_include_files("stdint.h" HAVE_STDINT_H) if(HAVE_STDINT_H) add_definitions(-DHAVE_STDINT_H) + else() + check_include_files("sys/types.h" HAVE_SYS_TYPES_H) + if(HAVE_SYS_TYPES_H) + add_definitions(-DHAVE_SYS_TYPES_H) + endif() endif() include(CheckStdInt) include(CheckCXXCompilerFlag) @@ -113,7 +118,8 @@ macro(interop_config_compiler_and_linker) set(ENABLE_BIG_OBJ_FLAG "-Wa,-mbig-obj") endif() - set(flags_to_check "-Wno-eof-newline;-Wno-maybe-uninitialized;-Wno-strict-aliasing;-Wno-unused-function;-Wno-unused-parameter;-Wno-unnamed-type-template-args;-Wno-c++0x-compat;-Wno-error=c++0x-compat") + + set(flags_to_check "-Wno-non-template-friend;-Wno-eof-newline;-Wno-maybe-uninitialized;-Wno-strict-aliasing;-fno-strict-aliasing;-Wno-unused-function;-Wno-unused-parameter;-Wno-unnamed-type-template-args;-Wno-c++0x-compat;-Wno-error=c++0x-compat") foreach(flag ${flags_to_check}) string(TOUPPER ${flag} FLAG_NAME) string(REPLACE "-" "_" FLAG_NAME ${FLAG_NAME}) @@ -127,18 +133,19 @@ macro(interop_config_compiler_and_linker) endforeach() if(COMPILER_IS_GNUCC_OR_CLANG) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror ${_WNO_CPP0X_COMPAT} ${_WNO_ERROR_EQ_CPP0X_COMPAT} ${_WNO_EOF_NEWLINE}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror ${_WNO_CPP0X_COMPAT} ${_WNO_ERROR_EQ_CPP0X_COMPAT} ${_WNO_EOF_NEWLINE} ${_WNO_NON_TEMPLATE_FRIEND}") elseif(MSVC) # Visual Studio Complains about not being able to create an assignment operator and copy constructor # -wd4511 and -wd4512 disable these pointless warnings - # Add big object support to Windows Compilers - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX -wd4511 -wd4512") + # -wd4127: conditional expression is constant + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX -wd4511 -wd4512 -wd4127") + # TODO: remove all C4127 in headers add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS) elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") if(WIN32) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror ${_WNO_CPP0X_COMPAT} ${_WNO_ERROR_EQ_CPP0X_COMPAT}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror ${_WNO_CPP0X_COMPAT} ${_WNO_ERROR_EQ_CPP0X_COMPAT} ${_WNO_NON_TEMPLATE_FRIEND}") endif() endif() if(NOT ENABLE_BACKWARDS_COMPATIBILITY) diff --git a/cmake/Modules/CheckIsNaN.cmake b/cmake/Modules/CheckIsNaN.cmake index 7b6d0ece0..8fadb2908 100644 --- a/cmake/Modules/CheckIsNaN.cmake +++ b/cmake/Modules/CheckIsNaN.cmake @@ -8,8 +8,8 @@ CHECK_CXX_SOURCE_COMPILES( "#include \nint main() { bool a = std::isnan(0e0); return 0; }\n" HAVE_STD_ISNAN) -if(HAVE_STD_ISNAN) - add_definitions(-DHAVE_STD_ISNAN) +if(NOT HAVE_STD_ISNAN) + add_definitions(-DHAVE_NO_STD_ISNAN) return() endif() diff --git a/cmake/Modules/FindCSharp.cmake b/cmake/Modules/FindCSharp.cmake index ec392bde3..556246fb3 100644 --- a/cmake/Modules/FindCSharp.cmake +++ b/cmake/Modules/FindCSharp.cmake @@ -53,15 +53,19 @@ endif( ) if( CSHARP_DOTNET_FOUND ) - if(CMAKE_SIZEOF_VOID_P EQUAL "4") + if(CMAKE_SIZEOF_VOID_P EQUAL "4" OR FORCE_X86) set( CSHARP_PLATFORM "x86" CACHE STRING "C# target platform: x86, x64, anycpu, or itanium" ) elseif( CMAKE_SIZEOF_VOID_P EQUAL "8" ) set( CSHARP_PLATFORM "x64" CACHE STRING "C# target platform: x86, x64, anycpu, or itanium" ) else() - message(FATAL_ERROR "Only 32-bit and 64-bit are supported") + message(FATAL_ERROR "Only 32-bit and 64-bit are supported: ${CMAKE_SIZEOF_VOID_P}") endif() elseif( CSHARP_MONO_FOUND ) - set( CSHARP_PLATFORM "anycpu" CACHE STRING "C# target platform: x86, x64, anycpu, or itanium" ) + if(CMAKE_SIZEOF_VOID_P EQUAL "4" OR FORCE_X86) + set( CSHARP_PLATFORM "x86" CACHE STRING "C# target platform: x86, x64, anycpu, or itanium" ) + else() + set( CSHARP_PLATFORM "anycpu" CACHE STRING "C# target platform: x86, x64, anycpu, or itanium" ) + endif() endif() diff --git a/cmake/Modules/FindGMock.cmake b/cmake/Modules/FindGMock.cmake index e60867591..1bf18f6fc 100644 --- a/cmake/Modules/FindGMock.cmake +++ b/cmake/Modules/FindGMock.cmake @@ -68,6 +68,9 @@ if(NOT GMOCK_INCLUDE_DIR OR NOT GMOCK_LIBRARY OR NOT GMOCK_MAIN_LIBRARY) else() set(gtest_force_shared_crt OFF) endif() + if(FORCE_X86) + set(EXTRA_FLAGS " -m32") + endif() set(GMOCK_PREFIX ${CMAKE_BINARY_DIR}/external/gtest) include(ExternalProject) ExternalProject_Add( @@ -75,7 +78,7 @@ if(NOT GMOCK_INCLUDE_DIR OR NOT GMOCK_LIBRARY OR NOT GMOCK_MAIN_LIBRARY) PREFIX ${GMOCK_PREFIX} GIT_REPOSITORY https://github.com/google/googlemock.git GIT_TAG release-1.7.0 - CMAKE_ARGS -DCMAKE_CXX_FLAGS=-DGMOCK_USE_OWN_TR1_TUPLE=${USE_OWN_TR1_TUPLE} + CMAKE_ARGS "-DCMAKE_CXX_FLAGS=-DGMOCK_USE_OWN_TR1_TUPLE=${USE_OWN_TR1_TUPLE}${EXTRA_FLAGS}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${GMOCK_PREFIX}/lib64 -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG=${GMOCK_PREFIX}/lib64 diff --git a/cmake/Modules/FindGTest.cmake b/cmake/Modules/FindGTest.cmake index 0ea9dddbb..469c87ce9 100644 --- a/cmake/Modules/FindGTest.cmake +++ b/cmake/Modules/FindGTest.cmake @@ -79,18 +79,22 @@ if(NOT GTEST_INCLUDE_DIR OR NOT GTEST_LIBRARY OR NOT GTEST_MAIN_LIBRARY OR NOT G set(USE_OWN_TR1_TUPLE 1) endif() if(FORCE_SHARED_CRT) + message(STATUS "Enable Shared CRT for GTest") set(gtest_force_shared_crt ON) else() set(gtest_force_shared_crt OFF) endif() set(GTEST_PREFIX ${CMAKE_BINARY_DIR}/external/gtest) + if(FORCE_X86) + set(EXTRA_FLAGS " -m32") + endif() include(ExternalProject) ExternalProject_Add( gtest PREFIX ${GTEST_PREFIX} GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG release-1.7.0 - CMAKE_ARGS -DCMAKE_CXX_FLAGS=-DGTEST_USE_OWN_TR1_TUPLE=${USE_OWN_TR1_TUPLE} + CMAKE_ARGS "-DCMAKE_CXX_FLAGS=-DGTEST_USE_OWN_TR1_TUPLE=${USE_OWN_TR1_TUPLE}${EXTRA_FLAGS}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${GTEST_PREFIX}/lib64 -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG=${GTEST_PREFIX}/lib64 diff --git a/cmake/package.nuspec.in b/cmake/package.nuspec.in index 5f4298d50..7a179ce56 100644 --- a/cmake/package.nuspec.in +++ b/cmake/package.nuspec.in @@ -24,7 +24,9 @@ native, Illumina, InterOp, C++, C#, @PLATFORM@, @CSHARP_TYPE@ - @NUGET_FILE_LIST@ + + + diff --git a/cmake/package.targets b/cmake/package.targets index ed28b2126..6e1127eae 100644 --- a/cmake/package.targets +++ b/cmake/package.targets @@ -1,8 +1,8 @@ - + - %(RecursiveDir)%(FileName)%(Extension) + %(FileName)%(Extension) PreserveNewest diff --git a/cmake/version.rc.in b/cmake/version.rc.in new file mode 100644 index 000000000..52da2fb89 --- /dev/null +++ b/cmake/version.rc.in @@ -0,0 +1,50 @@ +/* Illumina InterOp Library + * + * Version information for MSVC DLLs + */ + +#define VS_FF_DEBUG 0x1L +#define VS_VERSION_INFO 0x1L +#define VS_FFI_FILEFLAGSMASK 0x17L +#define VER_PRIVATEBUILD 0x0L +#define VER_PRERELEASE 0x0L +#define VOS__WINDOWS32 0x4L +#define VFT_DLL 0x2L +#define VFT2_UNKNOWN 0x0L + +#ifndef DEBUG +#define VER_DEBUG 0 +#else +#define VER_DEBUG VS_FF_DEBUG +#endif + + +VS_VERSION_INFO VERSIONINFO + FILEVERSION @VERSION_LIST@,0 + PRODUCTVERSION @VERSION_LIST@,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK + FILEFLAGS VER_DEBUG + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "Illumina InterOp Library\0" + VALUE "ProductVersion", "@VERSION@\0" + VALUE "FileVersion", "@VERSION@\0" + VALUE "InternalName", "@LIB_NAME@\0" + VALUE "ProductName", "Illumina InterOp Library: @LIB_NAME@\0" + VALUE "CompanyName", "Illumina, Inc.\0" + VALUE "LegalCopyright", "Copyright (C) 2015-2016\0" + VALUE "Licence", "GPL v3\0" + VALUE "Info", "https://github.com/Illumina/interop\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END + diff --git a/docs/src/changes.md b/docs/src/changes.md index c1e693156..c1de3f916 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -1,5 +1,21 @@ # Changes {#changes} +## v1.0.14 + +Date | Description +---------- | ----------- +2016-12-05 | Added regression tests for imaging +2016-12-05 | Added unit tests for format coverage +2016-11-17 | Improved performance of InterOp calculations for C++11 enabled version +2016-11-14 | Added version to Windows DLLs +2016-11-14 | Added regression tests for indexing +2016-11-09 | Added unit tests for clearing logic +2016-11-07 | Added regression tests for analysis plots +2016-11-01 | Improved performance of InterOp loading +2016-10-26 | Added path class to handle RTA output filenames +2016-09-22 | Added regression tests for summary + + ## v1.0.13 Date | Description diff --git a/docs/src/contribute.md b/docs/src/contribute.md index d024b9462..440f2790e 100644 --- a/docs/src/contribute.md +++ b/docs/src/contribute.md @@ -79,3 +79,12 @@ Linux: ~~~~~~~~~{.sh} ln -s tools/hooks/pre-commit.sh .git/hooks/pre-commit ~~~~~~~~~ + +## Debugging Unit Tests + +The unit tests use the Google Test Framework. The following flags may help with debugging: + + - `--gtest_catch_exceptions=0`: Ensure GTest throws an exception + - `--gtest_break_on_failure`: Ensure GTest stopps on the first failure + + diff --git a/docs/src/install.md b/docs/src/install.md index ba497d84e..26a14c294 100644 --- a/docs/src/install.md +++ b/docs/src/install.md @@ -27,7 +27,7 @@ how to build the source. Carefully consider the prerequisites before trying to b The following are the minimum requirements to build the source: - C/C++ Compiler (C++98) - - Windows: [MinGW], [Cygwin], Microsoft Visual C++, [Express] + - Windows: [MinGW] 6.x or later, [Cygwin], Microsoft Visual C++, [Express] - Linux: GCC, CLang - Mac OSX: CLang - [CMake] Version 3.2 or later diff --git a/interop/constants/enum_description.h b/interop/constants/enum_description.h new file mode 100644 index 000000000..fbe177956 --- /dev/null +++ b/interop/constants/enum_description.h @@ -0,0 +1,75 @@ +/** Encapsulate an enum and a string description + * + * @file + * @date 12/5/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#pragma once +#include +#include +#include "interop/constants/enums.h" + + +namespace illumina { namespace interop { namespace constants +{ + /** Encapsulates an enum and a string description + */ + template + class enum_description + { + public: + /** Type of the enum */ + typedef Enum enum_t; + + public: + /** Constructor */ + enum_description() : m_value(static_cast(constants::Unknown)){} + /** Constructor + * + * @param val enum value + * @param description enum description + */ + enum_description(const enum_t val, const std::string& description) : m_value(val), m_description(description){} + /** Constructor + * + * @param pair enum value/description pair + */ + enum_description(const std::pair& pair) : + m_value(pair.first), m_description(pair.second){} + + public: + /** Get the value of the enum + * + * @return enum value + */ + enum_t value()const + { + return m_value; + } + /** Get the description of the enum + * + * @return enum description + */ + const std::string& description()const + { + return m_description; + } + /** Implicit conversion operator + * + * @return enum value + */ + operator enum_t()const + { + return m_value; + } + + private: + Enum m_value; + std::string m_description; + }; +}}} + + + diff --git a/interop/constants/enums.h b/interop/constants/enums.h index c597ef2a6..206b03aab 100644 --- a/interop/constants/enums.h +++ b/interop/constants/enums.h @@ -12,6 +12,11 @@ #include #include #include "interop/util/cstdint.h" + +/** Sentinel for an unknown enum type */ +#define INTEROP_UNKNOWN 0x100 + + /** Enumeration of specific features that can belong to a metric * * @note This macro requires the macro INTEROP_TUPLE2 to be defined before use @@ -194,6 +199,27 @@ INTEROP_TUPLE1(Black),\ INTEROP_TUPLE1(UnknownColor) +/** Enumeration of plot types + * + * @note This macro requires the macro INTEROP_TUPLE1 to be defined before use + * @see illumina::interop::constants::plot_types + */ +#define INTEROP_ENUM_PLOT_TYPES \ + /** Spatial plot by tile */\ + INTEROP_TUPLE1(FlowcellPlot),\ + /** Plot data aggrergated by cycle */\ + INTEROP_TUPLE1(ByCyclePlot),\ + /** Spatial plot aggrergated by lane */\ + INTEROP_TUPLE1(ByLanePlot),\ + /** Histogram of q-scores */\ + INTEROP_TUPLE1(QHistogramPlot),\ + /** Heatmap of q-scores by cycle */\ + INTEROP_TUPLE1(QHeatmapPlot),\ + /** Indexing tab plot */\ + INTEROP_TUPLE1(SampleQCPlot),\ + /** Unknown plot type */\ + INTEROP_TUPLE1(UnknownPlotType) + /** Enumeration of bar plot options * * @note This macro requires the macro INTEROP_TUPLE1 to be defined before use @@ -206,8 +232,11 @@ INTEROP_TUPLE1(Shifted),\ INTEROP_TUPLE1(UnknownBarPlotOption) -/** Sentinel for an unknown enum type */ -#define INTEROP_UNKNOWN 0xff +///////////////////////////////////////////////////////////////////////////////////////////////////// +// Define x-macros +///////////////////////////////////////////////////////////////////////////////////////////////////// + + /** This temp macro converts a enum/description pair into the an enum (first element of the pair) */ #define INTEROP_TUPLE4(X, Y, Z, A) X /** This temp macro converts a enum/description pair into the an enum (first element of the pair) */ @@ -279,63 +308,14 @@ namespace illumina { namespace interop { namespace constants /** Options describing metric features */ enum metric_feature_type { - INTEROP_ENUM_METRIC_FEATURE_TYPE + INTEROP_ENUM_METRIC_FEATURE_TYPE = INTEROP_UNKNOWN }; - /** Encapsulates an enum and a string description - */ - template - class enum_description + /** Options for a bar plot */ + enum plot_types { - public: - /** Type of the enum */ - typedef Enum enum_t; - - public: - /** Constructor */ - enum_description() : m_value(static_cast(constants::Unknown)){} - /** Constructor - * - * @param val enum value - * @param description enum description - */ - enum_description(const enum_t val, const std::string& description) : m_value(val), m_description(description){} - /** Constructor - * - * @param pair enum value/description pair - */ - enum_description(const std::pair& pair) : - m_value(pair.first), m_description(pair.second){} - - public: - /** Get the value of the enum - * - * @return enum value - */ - enum_t value()const - { - return m_value; - } - /** Get the description of the enum - * - * @return enum description - */ - const std::string& description()const - { - return m_description; - } - /** Implicit conversion operator - * - * @return enum value - */ - operator enum_t()const - { - return m_value; - } - - private: - Enum m_value; - std::string m_description; + INTEROP_ENUM_PLOT_TYPES = INTEROP_UNKNOWN }; + }}} #undef INTEROP_TUPLE1 diff --git a/interop/constants/typedefs.h b/interop/constants/typedefs.h index 055069ba5..2e36a56db 100644 --- a/interop/constants/typedefs.h +++ b/interop/constants/typedefs.h @@ -11,7 +11,8 @@ #include "interop/constants/enums.h" -namespace illumina { namespace interop { namespace constants { +namespace illumina { namespace interop { namespace constants +{ /** Define base type for tile metrics */ typedef constant_type base_tile_t; diff --git a/interop/external/rapidxml.hpp b/interop/external/rapidxml.hpp index 3c40d2d6b..21fba220e 100644 --- a/interop/external/rapidxml.hpp +++ b/interop/external/rapidxml.hpp @@ -657,6 +657,8 @@ namespace rapidxml xml_base() : m_name(0) , m_value(0) + , m_name_size(0) + , m_value_size(0) , m_parent(0) { } @@ -807,7 +809,7 @@ namespace rapidxml //! Constructs an empty attribute with the specified type. //! Consider using memory_pool of appropriate xml_document if allocating attributes manually. - xml_attribute() + xml_attribute() : m_prev_attribute(0), m_next_attribute(0) { } @@ -901,7 +903,11 @@ namespace rapidxml xml_node(node_type type) : m_type(type) , m_first_node(0) + , m_last_node(0) , m_first_attribute(0) + , m_last_attribute(0) + , m_prev_sibling(0) + , m_next_sibling(0) { } diff --git a/interop/io/format/abstract_metric_format.h b/interop/io/format/abstract_metric_format.h index ca4b2329d..21d89683c 100644 --- a/interop/io/format/abstract_metric_format.h +++ b/interop/io/format/abstract_metric_format.h @@ -57,6 +57,13 @@ namespace illumina { namespace interop { namespace io virtual void read_metrics(std::istream& in, model::metric_base::metric_set& metric_set, const size_t file_size)=0; + /** Read only the header of a metric set + * + * @param in input stream + * @param metric_set destination set of metrics + * @return number of bytes read + */ + virtual size_t read_header(std::istream& in, model::metric_base::metric_set& metric_set)=0; /** Write a metric record to the given output stream * @@ -78,6 +85,11 @@ namespace illumina { namespace interop { namespace io * @return version number */ virtual ::int16_t version() const = 0; + /** Is the format a multi-record format + * + * @return true if multiple records make up a single metric + */ + virtual bool is_multi_record() const = 0; }; }}} diff --git a/interop/io/format/default_layout.h b/interop/io/format/default_layout.h index d8c435b6b..36b8387e3 100644 --- a/interop/io/format/default_layout.h +++ b/interop/io/format/default_layout.h @@ -54,6 +54,30 @@ namespace illumina { namespace interop { namespace io stream_map(stream, record_size); return record_size; } + /** Skip inserting this metric into the metric set + * + * This function was originally added to skip control records in tile metrics. + * + * @param metric metric to check + * @return true, if the metric id is 0 + */ + template + static bool skip_metric(const Metric& metric) + { + return metric.id() == 0; + } + /** Skip inserting this metric into the metric set + * + * This function was originally added to skip control records in tile metrics. + * + * @param metric metric to check + * @return true, if the metric id is 0 + */ + template + static bool is_valid(const LayoutId& id) + { + return id.is_valid(); + } }; }}} diff --git a/interop/io/format/map_io.h b/interop/io/format/map_io.h index 510695563..85cda9a8a 100644 --- a/interop/io/format/map_io.h +++ b/interop/io/format/map_io.h @@ -77,6 +77,29 @@ namespace illumina { namespace interop { namespace io return sizeof(ReadType); } + /** Read a string from the given input stream + * + * @param in input stream + * @param val destination string + * @return number of characters read + */ + inline std::streamsize stream_map(std::istream &in, std::string &val) + { + read_binary(in, val, ""); + return in.gcount(); + } + /** Read a string from the given input stream + * + * @param in input stream + * @param val destination string + * @return number of characters read + */ + inline std::streamsize stream_map(char*& in, std::string &val) + { + read_binary(in, val); + return val.size(); + } + /** Helper to read an array */ template @@ -278,6 +301,20 @@ namespace illumina { namespace interop { namespace io return out.tellp(); } + /** Write a string to the given output stream + * + * TODO: create more efficient buffered version + * + * @param out output stream + * @param str source string + * @return number of bytes written to the stream + */ + inline std::streamsize stream_map(std::ostream &out, const std::string &str) + { + write_binary(out, str); + return out.tellp(); + } + /** Write an array of values of type ReadType to the given output stream * * TODO: create more efficient buffered version @@ -290,6 +327,9 @@ namespace illumina { namespace interop { namespace io template std::streamsize stream_map(std::ostream &out, const ValueType &vals, const size_t n) { + INTEROP_ASSERT(util::length_of(vals) >= n); + if(util::length_of(vals) < n) + INTEROP_THROW(bad_format_exception, "Write bug: Number of values is less than expected!"); for (size_t i = 0; i < n; i++) { WriteType write_val = static_cast(vals[i]); @@ -311,6 +351,9 @@ namespace illumina { namespace interop { namespace io template std::streamsize stream_map(std::ostream &out, const ValueType &vals, const size_t offset, const size_t n) { + INTEROP_ASSERT(util::length_of(vals) >= (offset+n)); + if(util::length_of(vals) < (offset+n)) + INTEROP_THROW(bad_format_exception, "Write bug: Number of values is less than expected!"); for (size_t i = 0; i < n; i++) { WriteType write_val = static_cast(vals[offset + i]); @@ -322,7 +365,7 @@ namespace illumina { namespace interop { namespace io /** Placeholder that does nothing */ template - void map_resize(const std::vector &, size_t) + void map_resize(const Layout &, size_t) { } @@ -332,7 +375,7 @@ namespace illumina { namespace interop { namespace io * @param n number of elements */ template - void map_resize(std::vector &layout, const size_t n) + void map_resize(Layout &layout, const size_t n) { layout.resize(n); } diff --git a/interop/io/format/metric_format.h b/interop/io/format/metric_format.h index d3036843d..02815a2b3 100644 --- a/interop/io/format/metric_format.h +++ b/interop/io/format/metric_format.h @@ -32,7 +32,8 @@ namespace illumina { namespace interop { namespace io { private: typedef typename Metric::id_t id_t; - typedef std::map offset_map_t; + typedef model::metric_base::metric_set metric_set_t; + typedef typename metric_set_t::offset_map_t offset_map_t; public: /** Define the metric type */ typedef Metric metric_t; @@ -69,6 +70,19 @@ namespace illumina { namespace interop { namespace io write_binary(out, metric_id); Layout::map_stream(out, metric, header, false); } + /** Read the header into a metric set + * + * @param in input stream + * @param metric_set destination set of metrics + * @return number of bytes read + */ + size_t read_header(std::istream& in, model::metric_base::metric_set& metric_set) + { + const size_t version_byte_size = 1; + const std::streampos beg = in.tellg(); + read_header_impl(in, metric_set); + return static_cast(in.tellg()-beg)+version_byte_size; + } /** Read all the metrics into a metric set * @@ -76,10 +90,10 @@ namespace illumina { namespace interop { namespace io * @param metric_set destination set of metrics * @param file_size size of the file */ - void read_metrics(std::istream& in, model::metric_base::metric_set& metric_set, const size_t file_size) + void read_metrics(std::istream& in, metric_set_t& metric_set, const size_t file_size) { - const std::streamsize record_size = read_header(in, metric_set); - offset_map_t metric_offset_map(metric_set.offset_map()); + const std::streamsize record_size = read_header_impl(in, metric_set); + offset_map_t& metric_offset_map = metric_set.offset_map(); metric_t metric(metric_set); if(file_size > 0 && !Layout::MULTI_RECORD) { @@ -99,11 +113,10 @@ namespace illumina { namespace interop { namespace io } catch(const incomplete_file_exception& ex) { - metric_set.resize(metric_offset_map.size()); + metric_set.trim(metric_offset_map.size()); throw ex; } } - metric_set.resize(metric_offset_map.size()); } else { @@ -112,6 +125,7 @@ namespace illumina { namespace interop { namespace io read_record(in, metric_set, metric_offset_map, metric, record_size); } } + metric_set.trim(metric_offset_map.size()); } /** Read a metric set from the given input stream * @@ -119,7 +133,7 @@ namespace illumina { namespace interop { namespace io * @param header metric set header * @return number of bytes in the record */ - std::streamsize read_header(std::istream &in, header_t &header) + std::streamsize read_header_impl(std::istream &in, header_t &header) { // TODO: optimize header reading with block read if (in.fail()) @@ -186,6 +200,14 @@ namespace illumina { namespace interop { namespace io { return static_cast< ::int16_t >(Layout::VERSION); } + /** Is the format a multi-record format + * + * @return true if multiple records make up a single metric + */ + bool is_multi_record() const + { + return Layout::MULTI_RECORD > 0; + } private: static bool test_stream(std::istream& in, @@ -216,7 +238,9 @@ namespace illumina { namespace interop { namespace io const std::streamsize read_byte_count = read_binary_with_count (in, id); if(!test_stream(in, metric_offset_map, read_byte_count, record_size)) return; std::streamsize count=read_byte_count; - if (id.is_valid(metric_t::CHECK_TILE_ID)) + if (Layout::is_valid(id)) + // TODO: Refactor tile metrics to move record type into layout id, then we can remove skip_metric, + // simplifiy all this logic { metric.set_base(id);// TODO replace with static call if (metric_offset_map.find(metric.id()) == metric_offset_map.end()) @@ -225,7 +249,8 @@ namespace illumina { namespace interop { namespace io if(offset>= metric_set.size()) metric_set.resize(offset+1); metric_set.at(offset).set_base(id); count += Layout::map_stream(in, metric_set.at(offset), metric_set, true); - if(metric_set.at(offset).id()==0)//Avoid adding control lanes in tile metrics + if(!test_stream(in, metric_offset_map, count, record_size)) return; + if(Layout::skip_metric(metric_set.at(offset)))//Avoid adding control lanes in tile metrics { metric_set.resize(offset); } diff --git a/interop/io/format/metric_format_factory.h b/interop/io/format/metric_format_factory.h index c32db4d78..f6f56532c 100644 --- a/interop/io/format/metric_format_factory.h +++ b/interop/io/format/metric_format_factory.h @@ -7,8 +7,8 @@ */ #pragma once -#include #include +#include "interop/util/map.h" #include "interop/util/assert.h" #include "interop/io/format/abstract_metric_format.h" #include "interop/util/unique_ptr.h" @@ -68,7 +68,7 @@ namespace illumina { namespace interop { namespace io /** Define a unique pointer to a metric format */ typedef stdbp::unique_ptr metric_format_pointer; /** Define a map between format version and the format */ - typedef std::map metric_format_map; + typedef INTEROP_UNORDERED_MAP(int, metric_format_pointer) metric_format_map; /** Constructor * diff --git a/interop/io/format/stream_util.h b/interop/io/format/stream_util.h index 4a7de8b43..8a74761fb 100644 --- a/interop/io/format/stream_util.h +++ b/interop/io/format/stream_util.h @@ -134,8 +134,9 @@ namespace illumina { namespace interop { namespace io * * @param in input stream * @param str string buffer + * @param default_val default value for an empty string */ - inline void read_binary(std::istream &in, std::string &str) + inline void read_binary(std::istream &in, std::string &str, const std::string& default_val) { const size_t length = read_binary< ::uint16_t >(in); if (in.fail()) INTEROP_THROW(incomplete_file_exception, "No more data after length"); @@ -144,7 +145,7 @@ namespace illumina { namespace interop { namespace io str.assign(length, ' '); read_binary(in, const_cast(str.c_str()), length); } - else str = "NA"; + else str = default_val; } /** Write an array of data to an output stream diff --git a/interop/io/layout/base_metric.h b/interop/io/layout/base_metric.h index e49f5f861..ebe6c772f 100644 --- a/interop/io/layout/base_metric.h +++ b/interop/io/layout/base_metric.h @@ -19,6 +19,7 @@ #include #include "interop/util/static_assert.h" #include "interop/util/cstdint.h" +#include "interop/constants/typedefs.h" namespace illumina { namespace interop { namespace io { namespace layout { @@ -31,17 +32,24 @@ namespace illumina { namespace interop { namespace io { namespace layout * @note These classes are packed such that there is not padding. Their size reflects the accumulation of their * member fields. */ + template struct base_metric { + /** Lane integral type */ + typedef ::uint16_t lane_t; + /** Tile integral type */ + typedef T tile_t; /** Define a record size type */ typedef ::uint8_t record_size_t; + /** Define base type */ + typedef constants::base_tile_t base_t; /** Constructor * * @param lane_ lane number * @param tile_ tile number */ - base_metric(::uint16_t lane_ = 0, ::uint16_t tile_ = 0) : + base_metric(const lane_t lane_ = 0, const tile_t tile_ = 0) : lane(lane_), tile(tile_) { static_assert(sizeof(::uint16_t) == 2, "16-bit int not supported"); @@ -55,25 +63,25 @@ namespace illumina { namespace interop { namespace io { namespace layout template void set(const BaseMetric &metric) { - lane = static_cast< ::uint16_t >(metric.lane()); - tile = static_cast< ::uint16_t >(metric.tile()); + lane = static_cast< lane_t >(metric.lane()); + tile = static_cast< tile_t >(metric.tile()); } /** Test if the layout contains valid data * * @return true if data is valid */ - bool is_valid(const bool check_tile) const + bool is_valid() const { - return (!check_tile || tile > 0) && lane > 0; + return tile > 0 && lane > 0; } /** Lane number */ - ::uint16_t lane; + lane_t lane; /** Tile number */ - ::uint16_t tile; + tile_t tile; }; /** Base class for InterOp records that contain tile specific metrics per cycle @@ -83,16 +91,25 @@ namespace illumina { namespace interop { namespace io { namespace layout * @note These classes are packed such that there is not padding. Their size reflects the accumulation of their * member fields. */ - struct base_cycle_metric : base_metric + template + struct base_cycle_metric : base_metric { + /** Lane integral type */ + typedef typename base_metric::lane_t lane_t; + /** Tile integral type */ + typedef typename base_metric::tile_t tile_t; + /** Cycle intergral type */ + typedef ::uint16_t cycle_t; + /** Define base type */ + typedef constants::base_cycle_t base_t; /** Constructor * * @param lane_ lane number * @param tile_ tile number * @param cycle_ cycle number */ - base_cycle_metric(::uint16_t lane_ = 0, ::uint16_t tile_ = 0, ::uint16_t cycle_ = 0) : - base_metric(lane_, tile_), cycle(cycle_) + base_cycle_metric(const lane_t lane_ = 0, const tile_t tile_ = 0, const cycle_t cycle_ = 0) : + base_metric(lane_, tile_), cycle(cycle_) { } @@ -103,22 +120,22 @@ namespace illumina { namespace interop { namespace io { namespace layout template void set(const BaseMetric &metric) { - base_metric::set(metric); - cycle = static_cast< ::uint16_t >(metric.cycle()); + base_metric::set(metric); + cycle = static_cast< cycle_t >(metric.cycle()); } /** Test if the layout contains valid data * * @return true if data is valid */ - bool is_valid(const bool check_tile) const + bool is_valid() const { - return base_metric::is_valid(check_tile) && cycle > 0; + return base_metric::is_valid() && cycle > 0; } /** Cycle number */ - ::uint16_t cycle; + cycle_t cycle; }; /** Base class for InterOp records that contain tile specific metrics per read @@ -128,16 +145,25 @@ namespace illumina { namespace interop { namespace io { namespace layout * @note These classes are packed such that there is not padding. Their size reflects the accumulation * of their member fields. */ - struct base_read_metric : base_metric + template + struct base_read_metric : base_metric { + /** Lane integral type */ + typedef typename base_metric::lane_t lane_t; + /** Tile integral type */ + typedef typename base_metric::tile_t tile_t; + /** Read integral type */ + typedef ::uint16_t read_t; + /** Define base type */ + typedef constants::base_read_t base_t; /** Constructor * * @param lane_ lane number * @param tile_ tile number * @param read_ read number */ - base_read_metric(::uint16_t lane_ = 0, ::uint16_t tile_ = 0, ::uint16_t read_ = 0) : - base_metric(lane_, tile_), read(read_) + base_read_metric(const lane_t lane_ = 0, const tile_t tile_ = 0, const read_t read_ = 0) : + base_metric(lane_, tile_), read(read_) { } @@ -148,22 +174,22 @@ namespace illumina { namespace interop { namespace io { namespace layout template void set(const BaseMetric &metric) { - base_metric::set(metric); - read = static_cast< ::uint16_t >(metric.read()); + base_metric::set(metric); + read = static_cast< read_t >(metric.read()); } /** Test if the layout contains valid data * * @return true if data is valid */ - bool is_valid(const bool check_tile) const + bool is_valid() const { - return base_metric::is_valid(check_tile) && read > 0; + return base_metric::is_valid() && read > 0; } /** Read number */ - ::uint16_t read; + read_t read; }; #pragma pack() diff --git a/interop/io/metric_file_stream.h b/interop/io/metric_file_stream.h index e24dbdbcc..8b900a645 100644 --- a/interop/io/metric_file_stream.h +++ b/interop/io/metric_file_stream.h @@ -28,10 +28,13 @@ namespace illumina { namespace interop { namespace io * @return number of bytes required */ template - size_t compute_buffer_size(const MetricSet& metrics) + size_t compute_buffer_size(const MetricSet& metrics) throw(invalid_argument, bad_format_exception) { typedef typename MetricSet::metric_type metric_t; - return header_size(metrics, metrics.version()) + record_size(metrics, metrics.version()) * metrics.size(); + if(is_multi_record(metrics)) + throw invalid_argument("Format does not currently support computing the buffer size"); + return header_size(metrics, metrics.version()) + + size_of_record(metrics, metrics.version()) * metrics.size(); } /** Write the metric to a binary byte buffer * @@ -77,19 +80,19 @@ namespace illumina { namespace interop { namespace io * * @param buffer string holding a byte buffer * @param metrics metric set + * @param rebuild whether to rebuild the id map * @throw bad_format_exception * @throw incomplete_file_exception * @throw model::index_out_of_bounds_exception */ template - void read_interop_from_string(const std::string& buffer, MetricSet& metrics) throw + void read_interop_from_string(const std::string& buffer, MetricSet& metrics, const bool rebuild=true) throw (interop::io::bad_format_exception, interop::io::incomplete_file_exception, model::index_out_of_bounds_exception) { - detail::membuf sbuf(const_cast(buffer.c_str()), const_cast(buffer.c_str()) + buffer.length()); - std::istream in(&sbuf); - read_metrics(in, metrics, buffer.length()); + std::istringstream in(buffer); + read_metrics(in, metrics, buffer.length(), rebuild); } /** Read the binary InterOp file into the given metric set * @@ -100,16 +103,13 @@ namespace illumina { namespace interop { namespace io * @throw model::index_out_of_bounds_exception */ template - void read_interop_from_string(const std::vector< ::uint8_t>& buffer, MetricSet& metrics) throw + size_t read_header_from_string(const std::string& buffer, MetricSet& metrics) throw (interop::io::bad_format_exception, interop::io::incomplete_file_exception, model::index_out_of_bounds_exception) { - if(buffer.empty())return; - char* pbuffer =reinterpret_cast(const_cast< ::uint8_t* >(&buffer.front())); - detail::membuf sbuf(pbuffer, pbuffer + buffer.size()); - std::istream in(&sbuf); - read_metrics(in, metrics, buffer.size()); + std::istringstream in(buffer); + return read_header(in, metrics); } /** Read the binary InterOp file into the given metric set * @@ -142,26 +142,6 @@ namespace illumina { namespace interop { namespace io if(!fin.good()) INTEROP_THROW(file_not_found_exception, "File not found: " << file_name); read_metrics(fin, metrics, static_cast(file_size(file_name))); } - /** Check for the existence of the binary InterOp file into the given metric set - * - * @note The 'Out' suffix (parameter: use_out) is appended when we read the file. We excluded the Out in certain - * conditions when writing the file. - * - * @param run_directory file path to the run directory - * @param use_out use the copied version - */ - template - bool interop_exists(const std::string& run_directory, MetricSet&, const bool use_out=true) - throw(file_not_found_exception, - bad_format_exception, - incomplete_file_exception, - model::index_out_of_bounds_exception) - { - const std::string file_name = interop_filename(run_directory, use_out); - std::ifstream fin(file_name.c_str(), std::ios::binary); - if(!fin.good()) return false; - return true; - } /** Write the metric set to a binary InterOp file * * @note The 'Out' suffix (parameter: use_out) is appended when we read the file. We excluded the Out in certain @@ -182,7 +162,7 @@ namespace illumina { namespace interop { namespace io bad_format_exception, incomplete_file_exception) { - if(metrics.empty())return true; + if(metrics.empty() || metrics.version() == 0 )return true; const std::string file_name = interop_filename(run_directory, use_out); std::ofstream fout(file_name.c_str(), std::ios::binary); if(!fout.good())INTEROP_THROW(file_not_found_exception, "File not found: " << file_name); @@ -213,6 +193,43 @@ namespace illumina { namespace interop { namespace io if(!fout.good())INTEROP_THROW(file_not_found_exception, "File not found: " << file_name); write_metric_header(fout, header, version); } + /** Check for the existence of the binary InterOp file into the given metric set + * + * @note The 'Out' suffix (parameter: use_out) is appended when we read the file. We excluded the Out in certain + * conditions when writing the file. + * + * @param run_directory file path to the run directory + * @param use_out use the copied version + */ + template + bool interop_exists(const std::string& run_directory, MetricSet&, const bool use_out=true) + throw(file_not_found_exception, + bad_format_exception, + incomplete_file_exception, + model::index_out_of_bounds_exception) + { + const std::string file_name = interop_filename(run_directory, use_out); + std::ifstream fin(file_name.c_str(), std::ios::binary); + if(!fin.good()) return false; + return true; + } + /** List all possible InterOp file names + * + * @note The first filename is the legacy name + * + * @param files destination list of files + * @param run_directory file path to the run directory + * @param last_cycle last cycle to check + * @param use_out use the copied version + */ + template + void list_interop_filenames(std::vector& files, + const std::string& run_directory, + const bool use_out=true) + { + files.resize(1); + files[0] = interop_filename(run_directory, use_out); + } /** @} */ }}} diff --git a/interop/io/metric_stream.h b/interop/io/metric_stream.h index 7d036dcf5..32c049bc3 100644 --- a/interop/io/metric_stream.h +++ b/interop/io/metric_stream.h @@ -11,66 +11,13 @@ #include "interop/util/exception.h" #include "interop/model/model_exceptions.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/paths.h" #include "interop/util/filesystem.h" #include "interop/util/assert.h" #pragma once namespace illumina { namespace interop { namespace io { - - namespace detail - { - /** Append the InterOp directory to the run directory file path - * - * @param run_directory file path to the run directory - * @return file path to the InterOp directory - */ - inline std::string interop_directory_name(const std::string &run_directory) - { - return io::combine(run_directory, "InterOp"); - } - - /** Generate a file name from a run directory and the InterOp name - * - * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions - * when writing the file. - * - * @param prefix prefix name of the interop file - * @param suffix suffix name of the interop file - * @param use_out if true, append "Out" to the end of the filename - * @return file path to the InterOp directory - */ - inline std::string interop_basename(const std::string &prefix, - const std::string &suffix, - const bool use_out = true) - { - return prefix + "Metrics" + suffix + ((use_out) ? ("Out.bin") : (".bin")); - } - - /** Generate a file name from a run directory and the InterOp name - * - * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions - * when writing the file. - * - * @param run_directory file path to the run directory - * @param prefix prefix name of the interop file - * @param suffix suffix name of the interop file - * @param use_out if true, append "Out" to the end of the filename - * @return file path to the InterOp directory - */ - inline std::string interop_filename(const std::string &run_directory, - const std::string &prefix, - const std::string &suffix, - const bool use_out = true) - { - if (io::basename(run_directory) == interop_basename(prefix, suffix, use_out)) - return run_directory; - if (io::basename(run_directory) == "InterOp") - return io::combine(run_directory, interop_basename(prefix, suffix, use_out)); - return io::combine(interop_directory_name(run_directory), interop_basename(prefix, suffix, use_out)); - } - } - /** Generate a file name from a run directory and the metric type * * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions @@ -83,7 +30,7 @@ namespace illumina { namespace interop { namespace io template std::string interop_filename(const std::string &run_directory, bool use_out = true) { - return detail::interop_filename(run_directory, MetricType::prefix(), MetricType::suffix(), use_out); + return paths::interop_filename(run_directory, use_out); } /** Generate a file name from a run directory and the metric type @@ -97,7 +44,47 @@ namespace illumina { namespace interop { namespace io template std::string interop_basename(bool use_out = true) { - return detail::interop_basename(MetricType::prefix(), MetricType::suffix(), use_out); + return paths::interop_basename(use_out); + } + + /** Copy list of version numbers to output iterator + * + * @param oit destination iterator for versions + */ + template + void copy_versions(OutputIterator oit) + { + typedef metric_format_factory factory_t; + typedef typename factory_t::metric_format_map metric_format_map; + metric_format_map &format_map = factory_t::metric_formats(); + for(typename metric_format_map::const_iterator it = format_map.begin();it != format_map.end();++it, ++oit) + { + *oit = it->first; + } + } + /** Read the header from the stream + * + * @param in input stream + * @param metrics metric set + * @return number of bytes read from the stream + */ + template + size_t read_header(std::istream &in, MetricSet &metrics) + { + typedef typename MetricSet::metric_type metric_t; + typedef metric_format_factory factory_t; + typedef typename factory_t::metric_format_map metric_format_map; + metric_format_map &format_map = factory_t::metric_formats(); + if (!in.good()) INTEROP_THROW(incomplete_file_exception, "Empty file found"); + const int version = in.get(); + if (version == -1) INTEROP_THROW(incomplete_file_exception, "Empty file found"); + if (format_map.find(version) == format_map.end()) + INTEROP_THROW(bad_format_exception, "No format found to parse " << paths::interop_basename() + << " with version: " << version << " of " + << format_map.size() ); + INTEROP_ASSERT(format_map[version]); + metrics.set_version(static_cast< ::int16_t>(version)); + return format_map[version]->read_header(in, metrics); } /** Read the binary InterOp file into the given metric set @@ -105,9 +92,10 @@ namespace illumina { namespace interop { namespace io * @param in input stream * @param metrics metric set * @param file_size number of bytes in the file + * @param rebuild flag indicating whether to rebuild the lookup table */ template - void read_metrics(std::istream &in, MetricSet &metrics, const size_t file_size) + void read_metrics(std::istream &in, MetricSet &metrics, const size_t file_size, const bool rebuild=true) { typedef typename MetricSet::metric_type metric_t; typedef metric_format_factory factory_t; @@ -117,8 +105,9 @@ namespace illumina { namespace interop { namespace io const int version = in.get(); if (version == -1) INTEROP_THROW(incomplete_file_exception, "Empty file found"); if (format_map.find(version) == format_map.end()) - INTEROP_THROW(bad_format_exception, "No format found to parse " << - interop_basename() << " with version: " << version << " of " << format_map.size() ); + INTEROP_THROW(bad_format_exception, "No format found to parse " << paths::interop_basename() + << " with version: " << version << " of " + << format_map.size() ); INTEROP_ASSERT(format_map[version]); metrics.set_version(static_cast< ::int16_t>(version)); try @@ -127,10 +116,10 @@ namespace illumina { namespace interop { namespace io } catch(const incomplete_file_exception& ex) { - metrics.rebuild_index(); + if(rebuild)metrics.rebuild_index();// TODO: Speed up with hash map if using c++11 compiler throw ex; } - metrics.rebuild_index(); + if(rebuild)metrics.rebuild_index(); } /** Get the size of a single metric record @@ -155,28 +144,88 @@ namespace illumina { namespace interop { namespace io return format_map[version]->record_size(header); } + /** Get the size of a single metric record + * + * @param header header for metric + * @param version version of the format + * @return size of a single metric record + */ + template + size_t size_of_record(const HeaderType &header, + const ::int16_t version = MetricType::LATEST_VERSION) + { + typedef metric_format_factory factory_type; + typedef typename factory_type::metric_format_map metric_format_map; + metric_format_map &format_map = factory_type::metric_formats(); + + if (format_map.find(version) == format_map.end()) + INTEROP_THROW(bad_format_exception, "No format found to write file with version: " << + version << " of " << format_map.size()); + + INTEROP_ASSERT(format_map[version]); + return format_map[version]->record_size(header); + } + /** Test whether the format support multi-records + * + * @param metric_set header for metric + * @param version version of the format + * @return true if the format supports multiple records + */ + template + bool is_multi_record(const MetricSet &header, + ::int16_t version = -1) + { + typedef typename MetricSet::metric_type metric_t; + typedef metric_format_factory factory_type; + typedef typename factory_type::metric_format_map metric_format_map; + metric_format_map &format_map = factory_type::metric_formats(); + if(version < 1) version = header.version(); + + if (format_map.find(version) == format_map.end()) + INTEROP_THROW(bad_format_exception, "No format found to write file with version: " << + version << " of " << format_map.size()); + + INTEROP_ASSERT(format_map[version]); + return format_map[version]->is_multi_record(); + } + + /** Get the size of a metric file header * * @param header header for metric * @param version version of the format * @return size of metric file header */ - template - size_t header_size(const typename MetricType::header_type &header, - const ::int16_t version = MetricType::LATEST_VERSION) + template + size_t size_of_header(const Header &header, + const ::int16_t version=MetricType::LATEST_VERSION) { typedef metric_format_factory factory_type; typedef typename factory_type::metric_format_map metric_format_map; metric_format_map &format_map = factory_type::metric_formats(); - if (format_map.find(version) == format_map.end()) INTEROP_THROW(bad_format_exception, "No format found to write file with version: " << - version << " of " << format_map.size()); + version << " of " << format_map.size()); INTEROP_ASSERT(format_map[version]); return format_map[version]->header_size(header); } + /** Get the size of a metric file header + * + * @param header header for metric + * @param version version of the format + * @return size of metric file header + */ + template + size_t header_size(const MetricSet &header, + ::int16_t version = -1) + { + typedef typename MetricSet::metric_type metric_t; + if(version < 1) version = header.version(); + return size_of_header(header, version); + } + /** Write a metric to a binary InterOp file * * @param out output stream @@ -247,8 +296,8 @@ namespace illumina { namespace interop { namespace io INTEROP_ASSERT(format_map[version]); format_map[version]->write_metric_header(out, metrics); - for (typename MetricSet::metric_array_t::const_iterator it = metrics.metrics().begin(); - it != metrics.metrics().end(); it++) + for (typename MetricSet::const_iterator it = metrics.begin(); + it != metrics.end(); it++) format_map[version]->write_metric(out, *it, metrics); } }}} diff --git a/interop/io/paths.h b/interop/io/paths.h new file mode 100644 index 000000000..f84a34537 --- /dev/null +++ b/interop/io/paths.h @@ -0,0 +1,166 @@ +/** Interface for common file paths + * + * + * @file + * @date 10/25/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include "interop/util/filesystem.h" + +namespace illumina { namespace interop { namespace io +{ + + /** Location of files written to an RTA run folder + */ + class paths + { + public: + /** Get the path to the RTA configuration file + * + * @param version version of RTA + * @return config filename + */ + static std::string rta_config(const std::string &run_directory, const int version=3) + { + return io::combine(run_directory, rta_config(version)); + } + /** Get the path of the RunInfo + * + * @return path/to/RunInfo.xml + */ + static std::string run_info(const std::string &run_directory) + { + return io::combine(run_directory, run_info()); + } + /** Get the path of the RunParameters + * + * @param alternate return alternate filename + * @return path/to/RunParameters.xml + */ + static std::string run_parameters(const std::string &run_directory, const bool alternate=false) + { + return io::combine(run_directory, run_parameters(alternate)); + } + /** Get the name of the RunInfo + * + * @return RunInfo.xml + */ + static std::string run_info() + { + return "RunInfo.xml"; + } + /** Get the name of the RunParameters + * + * @param alternate return alternate filename + * @return RunParameters.xml (alternate: runParameters.xml) + */ + static std::string run_parameters(const bool alternate=false) + { + if(alternate) return "runParameters.xml"; + return "RunParameters.xml"; + } + /** Generate a file name from a run directory and the metric type + * + * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions + * when writing the file. + * + * @param run_directory file path to the run directory + * @param use_out if true, append "Out" to the end of the filename + * @return file path to the InterOp directory + */ + template + static std::string interop_filename(const std::string &run_directory, bool use_out = true) + { + return interop_filename(run_directory, MetricType::prefix(), MetricType::suffix(), use_out); + } + + /** Generate a file name from a run directory and the metric type + * + * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions + * when writing the file. + * + * @param use_out if true, append "Out" to the end of the filename + * @return file path to the InterOp directory + */ + template + static std::string interop_basename(bool use_out = true) + { + return interop_basename(MetricType::prefix(), MetricType::suffix(), use_out); + } + /** Get the path to the RTA configuration file + * + * @param version version of RTA + * @return config filename + */ + static std::string rta_config(const int version=3) + { + if(version == 3) return "RTA3.cfg"; + return "RTAConfiguration.xml"; + } + + private: + /** Generate a file name from a run directory and the InterOp name + * + * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions + * when writing the file. + * + * @param prefix prefix name of the interop file + * @param suffix suffix name of the interop file + * @param use_out if true, append "Out" to the end of the filename + * @return file path to the InterOp directory + */ + static std::string interop_basename(const std::string &prefix, + const std::string &suffix, + const bool use_out = true) + { + return prefix + "Metrics" + suffix + ((use_out) ? ("Out.bin") : (".bin")); + } + + /** Generate a file name from a run directory and the InterOp name + * + * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions + * when writing the file. + * + * @param run_directory file path to the run directory + * @param prefix prefix name of the interop file + * @param suffix suffix name of the interop file + * @param use_out if true, append "Out" to the end of the filename + * @return file path to the InterOp directory + */ + static std::string interop_filename(const std::string &run_directory, + const std::string &prefix, + const std::string &suffix, + const bool use_out = true) + { + if (io::basename(run_directory) == interop_basename(prefix, suffix, use_out)) + return run_directory; + if (io::basename(run_directory) == "InterOp") + return io::combine(run_directory, interop_basename(prefix, suffix, use_out)); + return io::combine(interop_directory_name(run_directory), interop_basename(prefix, suffix, use_out)); + } + /** Append the InterOp directory to the run directory file path + * + * @param run_directory file path to the run directory + * @return file path to the InterOp directory + */ + static std::string interop_directory_name(const std::string &run_directory) + { + return io::combine(run_directory, "InterOp"); + } + /** Generate the proper name of a cycle folder from a given cycle + * + * @param cycle cycle number + * @return cycle folder name + */ + static std::string cycle_folder(const size_t cycle) + { + return "C" + util::lexical_cast(cycle)+".1"; + } + }; + +}}} + + diff --git a/interop/io/plot/gnuplot.h b/interop/io/plot/gnuplot.h index f7b0e9de6..df99765dd 100644 --- a/interop/io/plot/gnuplot.h +++ b/interop/io/plot/gnuplot.h @@ -12,6 +12,7 @@ #include "interop/model/plot/bar_point.h" #include "interop/model/plot/candle_stick_point.h" #include "interop/model/plot/plot_data.h" +#include "interop/io/table/csv_format.h" namespace illumina { namespace interop { namespace io { namespace plot { @@ -37,21 +38,24 @@ namespace illumina { namespace interop { namespace io { namespace plot out << "unset key\n"; out << "unset tics\n"; out << "unset border\n"; + out << "set cbtics border in scale 0,0 mirror norotate autojustify\n"; out << "set cbrange [" << data.saxis().min() << ":" << data.saxis().max() << "]\n"; out << - "set palette defined (0 \"blueviolet\", 0.143 \"blue\", 0.286 \"aqua\", 0.429 \"lightgreen\", 0.572 \"limegreen\", 0.715 \"lime\", 0.858 \"yellow\", 1 \"orange\")\n"; + "set palette defined (0 \"#8A2BE2\", 0.143 \"blue\", 0.286 \"cyan\", 0.429 \"light-green\", 0.572 \"#32CD32\", 0.715 \"#00FF00\", 0.858 \"yellow\", 1 \"orange\")\n"; out << "set autoscale fix\n"; out << "set size ratio 2\n"; + out << "set yrange[:] reverse\n"; out << "plot \"-\" matrix with image" << "\n"; const size_t swath_count = data.column_count() / data.tile_count(); - for (size_t y = 0; y < data.tile_count(); ++y) + for (size_t y = 0; y < data.tile_count(); ++y) // Rows { - for (size_t x = 0; x < data.row_count(); ++x) + for (size_t x = 0; x < data.row_count(); ++x) // Column groups { - for (size_t s = 0; s < swath_count; ++s) + for (size_t s = 0; s < swath_count; ++s) // Columns { - out << " " << data.at(x, y + s * data.tile_count()); + out << " " << table::handle_nan(data.at(x, y + s * data.tile_count())); } + out << " nan"; } out << "\n"; } @@ -108,7 +112,7 @@ namespace illumina { namespace interop { namespace io { namespace plot out << data.at(0, y); for (size_t x = 1; x < data.row_count(); ++x) { - out << " " << data.at(x, y); + out << " " << table::handle_nan(data.at(x, y)); } out << "\n"; } @@ -204,7 +208,8 @@ namespace illumina { namespace interop { namespace io { namespace plot */ void write_axis(std::ostream &out, const model::plot::axis &axis, const char axis_label) { - out << "set " << axis_label << "range [" << axis.min() << " : " << axis.max() << " ]\n"; + out << "set " << axis_label << "range [" + << table::handle_nan(axis.min()) << " : " << table::handle_nan(axis.max()) << " ]\n"; if (axis.label() != "") out << "set " << axis_label << "label \"" << axis.label() << "\"\n"; } @@ -246,12 +251,12 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t i = 0; i < series.size(); ++i) { - out << series[i].x() << "\t"; - out << series[i].lower() << "\t"; - out << series[i].p25() << "\t"; - out << series[i].p50() << "\t"; - out << series[i].p75() << "\t"; - out << series[i].upper(); + out << table::handle_nan(series[i].x()) << "\t"; + out << table::handle_nan(series[i].lower()) << "\t"; + out << table::handle_nan(series[i].p25()) << "\t"; + out << table::handle_nan(series[i].p50()) << "\t"; + out << table::handle_nan(series[i].p75()) << "\t"; + out << table::handle_nan(series[i].upper()); out << std::endl; } } @@ -268,7 +273,7 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t i = 0; i < series.size(); ++i) { - out << series[i].x() << "\t" << series[i].y() << std::endl; + out << table::handle_nan(series[i].x()) << "\t" << table::handle_nan(series[i].y()) << std::endl; } } @@ -292,7 +297,9 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t i = 0; i < series.size(); ++i) { - out << series[i].x() << "\t" << series[i].y() << "\t" << series[i].width() << std::endl; + out << table::handle_nan(series[i].x()) << "\t" + << table::handle_nan(series[i].y()) << "\t" + << table::handle_nan(series[i].width()) << std::endl; } } diff --git a/interop/io/table/csv_format.h b/interop/io/table/csv_format.h index fc8001746..53953af58 100644 --- a/interop/io/table/csv_format.h +++ b/interop/io/table/csv_format.h @@ -7,6 +7,7 @@ */ #pragma once #include "interop/util/lexical_cast.h" +#include "interop/util/math.h" namespace illumina { namespace interop { namespace io { namespace table @@ -31,6 +32,103 @@ namespace illumina { namespace interop { namespace io { namespace table else values.push_back(util::lexical_cast(cell)); } } + /** Read delimited value from the input stream and cast to proper destination type + * + * @param in input stream + * @param value string value + * @param delim deliminator + * @return destintion value + */ + template + T read_value(std::istream& in, std::string& buf, const char delim=',') + { + std::getline(in, buf, delim); + return util::lexical_cast(buf); + } + /** Read delimited value from the input stream and cast to proper destination type + * + * @param in input stream + * @param dest destination type + * @param value string value + * @param delim deliminator + */ + template + void read_value(std::istream& in, T& dest, std::string& buf, const char delim=',') + { + std::getline(in, buf, delim); + dest = util::lexical_cast(buf); + } + /** Read a csv values into a preallocated buffer + * + * @param in input stream + * @param beg start of the buffer + * @param end end of the buffer + * @param delim deliminator + */ + template + void read_csv(std::istream& in, I beg, I end, const char delim=',') + { + std::string buf; + for(;beg != end;++beg) + { + read_value(in, *beg, buf, delim); + } + } + /** Ignore non-float values + * + * @todo: Make this work for any floating point type + * @param val non-float value + * @return value + */ + template + inline const T& handle_nan(const T& val) + { + return val; + } + /** Ensure we get consistent NaNs + * + * @param val float value + * @return float value + */ + inline float handle_nan(const float val) + { + if(std::isnan(val)) return std::numeric_limits::quiet_NaN(); + return val; + } + /** Ensure we get consistent NaNs + * + * @param val float value + * @return float value + */ + inline double handle_nan(const double val) + { + if(std::isnan(val)) return std::numeric_limits::quiet_NaN(); + return val; + } + /** Write a vector of values as a single in a CSV file + * + * @param out output stream + * @param values source vector of values + * @param beg start column offset + * @param last last column offset + */ + template + void write_csv(std::ostream& out, I beg, I end, const char eol, const size_t precision=10) + { + if(beg == end) return; + std::ios::fmtflags previous_state( out.flags() ); + out << handle_nan(*beg); + ++beg; + for(;beg != end;++beg) + { + if(precision>0) + out << "," << std::setprecision(precision) << handle_nan(*beg); + else + out << "," << handle_nan(*beg); + } + if(eol != '\0') out << eol; + out.flags(previous_state); + } /** Write a vector of values as a single in a CSV file * * @param out output stream @@ -44,13 +142,8 @@ namespace illumina { namespace interop { namespace io { namespace table if(values.empty())return; if(last == 0) last = values.size(); INTEROP_ASSERT(beg < values.size()); - out << values[beg]; INTEROP_ASSERT(last <= values.size()); - for(typename std::vector::const_iterator it=values.begin()+beg+1, end=values.begin()+last;it != end;++it) - { - out << "," << *it; - } - out << "\n"; + write_csv(out, values.begin()+beg, values.begin()+std::min(last, values.size()), '\n'); } }}}} diff --git a/interop/logic/metric/metric_value.h b/interop/logic/metric/metric_value.h index e0f84b94c..3aa15a4c4 100644 --- a/interop/logic/metric/metric_value.h +++ b/interop/logic/metric/metric_value.h @@ -207,9 +207,19 @@ namespace illumina { namespace interop { namespace logic { namespace metric case constants::BasePercent: return metric.percent_base(base); case constants::CorrectedIntensity: - return metric.corrected_int_all(base); + { + const ::uint16_t corrected_int_all = metric.corrected_int_all(base); + if (corrected_int_all == std::numeric_limits< ::uint16_t>::max()) + return std::numeric_limits::quiet_NaN(); + return static_cast(corrected_int_all); + } case constants::CalledIntensity: - return metric.corrected_int_called(base); + { + const ::uint16_t corrected_int_called = metric.corrected_int_called(base); + if (corrected_int_called == std::numeric_limits< ::uint16_t>::max()) + return std::numeric_limits::quiet_NaN(); + return static_cast(corrected_int_called); + } case constants::SignalToNoise: return metric.signal_to_noise(); case constants::PercentNoCall: @@ -223,6 +233,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric }; + /** Specialization for model::metrics::tile_metric * * Supports enums: DensityPF, Density, ClusterCount, ClusterCountPF, PercentAligned, PercentPhasing, diff --git a/interop/logic/metric/q_metric.h b/interop/logic/metric/q_metric.h index 6b68df5a4..4f66931f7 100644 --- a/interop/logic/metric/q_metric.h +++ b/interop/logic/metric/q_metric.h @@ -160,7 +160,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric */ inline size_t count_qvals(const model::metric_base::metric_set& q_metric_set) { - return q_metric_set.size() > 0 ? q_metric_set.metrics()[0].size() : 0; + return q_metric_set.size() > 0 ? q_metric_set.at(0).size() : 0; } /** Test whether the q-values are compressed * @@ -169,7 +169,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric */ inline size_t count_qvals( const model::metric_base::metric_set& q_metric_set) { - return q_metric_set.size() > 0 ? q_metric_set.metrics()[0].size() : 0; + return q_metric_set.size() > 0 ? q_metric_set.at(0).size() : 0; } /** Test whether the q-values are compressed * @@ -218,7 +218,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric inline size_t max_qval( const model::metric_base::metric_set& q_metric_set) { - return is_compressed(q_metric_set) ? q_metric_set.bins().back().upper() : count_qvals(q_metric_set); + return is_compressed(q_metric_set) ? q_metric_set.get_bins().back().upper() : count_qvals(q_metric_set); } /** Determine the maximum Q-value * @@ -227,7 +227,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric */ inline size_t max_qval(const model::metric_base::metric_set& q_metric_set) { - return is_compressed(q_metric_set) ? q_metric_set.bins().back().upper() : count_qvals(q_metric_set); + return is_compressed(q_metric_set) ? q_metric_set.get_bins().back().upper() : count_qvals(q_metric_set); } /** Determine the maximum Q-value * @@ -237,7 +237,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric inline size_t max_qval(const model::metric_base::metric_set& q_metric_set) { return is_compressed(q_metric_set) ? - static_cast(q_metric_set.bins().back().upper()) : count_qvals(q_metric_set); + static_cast(q_metric_set.get_bins().back().upper()) : count_qvals(q_metric_set); } /** Get the index for the given q-value * diff --git a/interop/logic/metric/tile_metric.h b/interop/logic/metric/tile_metric.h index a14e595b9..d52d7e1f0 100644 --- a/interop/logic/metric/tile_metric.h +++ b/interop/logic/metric/tile_metric.h @@ -60,7 +60,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric constants::tile_naming_method tile_naming_method_from_metric(const model::metric_base::metric_set& metric_set) { if(metric_set.size() == 0 ) return constants::UnknownTileNamingMethod; - return tile_naming_method_from_metric(metric_set.metrics()[0]); + return tile_naming_method_from_metric(metric_set.at(0)); } /** Tile number * diff --git a/interop/logic/plot/plot_flowcell_map.h b/interop/logic/plot/plot_flowcell_map.h index f3ab9920d..ded7c6855 100644 --- a/interop/logic/plot/plot_flowcell_map.h +++ b/interop/logic/plot/plot_flowcell_map.h @@ -36,6 +36,34 @@ namespace illumina { namespace interop { namespace logic { namespace plot throw(model::invalid_filter_option, model::invalid_metric_type, model::index_out_of_bounds_exception); + /** Plot a flowcell map + * + * @ingroup plot_logic + * @param metrics run metrics + * @param type specific metric value to plot by cycle + * @param options options to filter the data + * @param data output flowcell map + * @param buffer preallocated memory for data + * @param buffer_size size of the buffer + * @param id_buffer preallocated memory for tile ids + * @param id_buffer_size size of the buffer + */ + inline void plot_flowcell_map2(model::metrics::run_metrics& metrics, + const constants::metric_type type, + const model::plot::filter_options& options, + model::plot::flowcell_data& data, + float* buffer, + const size_t buffer_size, + ::uint32_t* id_buffer, + const size_t id_buffer_size) + throw(model::invalid_filter_option, + model::invalid_metric_type, + model::index_out_of_bounds_exception) + { + (void)buffer_size; + (void)id_buffer_size; + plot_flowcell_map(metrics, type, options, data, buffer, id_buffer); + } /** Plot a flowcell map * @@ -56,6 +84,34 @@ namespace illumina { namespace interop { namespace logic { namespace plot throw(model::invalid_filter_option, model::invalid_metric_type, model::index_out_of_bounds_exception); + /** Plot a flowcell map + * + * @ingroup plot_logic + * @param metrics run metrics + * @param metric_name specific metric value to plot by cycle + * @param options options to filter the data + * @param data output flowcell map + * @param buffer preallocated memory for data + * @param buffer_size size of the buffer + * @param id_buffer preallocated memory for tile ids + * @param id_buffer_size size of the buffer + */ + inline void plot_flowcell_map2(model::metrics::run_metrics& metrics, + const std::string& metric_name, + const model::plot::filter_options& options, + model::plot::flowcell_data& data, + float* buffer, + const size_t buffer_size, + ::uint32_t* id_buffer, + const size_t id_buffer_size) + throw(model::invalid_filter_option, + model::invalid_metric_type, + model::index_out_of_bounds_exception) + { + (void)buffer_size; + (void)id_buffer_size; + plot_flowcell_map(metrics, metric_name, options, data, buffer, id_buffer); + } /** List metric type names available for flowcell * diff --git a/interop/logic/plot/plot_qscore_heatmap.h b/interop/logic/plot/plot_qscore_heatmap.h index 63706a27b..9dd80d495 100644 --- a/interop/logic/plot/plot_qscore_heatmap.h +++ b/interop/logic/plot/plot_qscore_heatmap.h @@ -25,7 +25,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot void plot_qscore_heatmap(model::metrics::run_metrics& metrics, const model::plot::filter_options& options, model::plot::heatmap_data& data, - float* buffer=0) + float* buffer=0, + const size_t buffer_size=0) throw(model::index_out_of_bounds_exception, model::invalid_filter_option); /** Count number of rows for the heat map diff --git a/interop/logic/plot/plot_qscore_histogram.h b/interop/logic/plot/plot_qscore_histogram.h index ba2e49583..ae3230edb 100644 --- a/interop/logic/plot/plot_qscore_histogram.h +++ b/interop/logic/plot/plot_qscore_histogram.h @@ -5,6 +5,7 @@ * @version 1.0 * @copyright GNU Public License. */ +#pragma once #include "interop/model/run_metrics.h" #include "interop/model/plot/filter_options.h" diff --git a/interop/logic/summary/cycle_state_summary.h b/interop/logic/summary/cycle_state_summary.h index 9ff855aeb..d3f95f3e2 100644 --- a/interop/logic/summary/cycle_state_summary.h +++ b/interop/logic/summary/cycle_state_summary.h @@ -7,6 +7,7 @@ */ #pragma once +#include "interop/util/map.h" #include "interop/logic/summary/map_cycle_to_read.h" #include "interop/model/metric_base/metric_set.h" #include "interop/model/metrics/tile_metric.h" @@ -43,8 +44,8 @@ namespace illumina { namespace interop { namespace logic { namespace summary cycle_range_vector2d_t summary_by_lane_read(run.size(), std::vector(run.lane_count())); typedef model::metrics::tile_metric::id_t id_t; - typedef std::map cycle_range_tile_t; - typedef std::map max_tile_map_t; + typedef INTEROP_UNORDERED_MAP(id_t, cycle_range) cycle_range_tile_t; + typedef INTEROP_UNORDERED_MAP(id_t, size_t) max_tile_map_t; typedef typename max_tile_map_t::const_iterator const_max_tile_iterator; typedef std::vector cycle_range_by_read_tile_t; cycle_range_by_read_tile_t tmp(summary_by_lane_read.size()); @@ -61,10 +62,12 @@ namespace illumina { namespace interop { namespace logic { namespace summary const read_cycle &read = cycle_to_read[cycle_metric_it->cycle() - 1]; if (read.number == 0) continue; INTEROP_ASSERT((read.number - 1) < tmp.size()); - const id_t id = model::metric_base::base_metric::create_id(cycle_metric_it->lane(), cycle_metric_it->tile()); + + const id_t id = cycle_metric_it->tile_hash(); tmp[read.number - 1][id].update(cycle_metric_it->cycle()); typename max_tile_map_t::iterator it = tmp_by_tile.find(id); - if (it == tmp_by_tile.end()) tmp_by_tile[id] = cycle_metric_it->cycle(); + if (it == tmp_by_tile.end()) + tmp_by_tile[id] = cycle_metric_it->cycle(); else it->second = std::max(static_cast(cycle_metric_it->cycle()), it->second); } @@ -76,7 +79,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary const id_t id = model::metric_base::base_metric::create_id(tile_it->lane(), tile_it->tile()); for (size_t read_index = 0; read_index < tmp.size(); ++read_index) { - if (tmp[read_index].find(tile_it->id()) == tmp[read_index].end()) + if (tmp[read_index].find(tile_it->tile_hash()) == tmp[read_index].end()) { tmp[read_index][id].update(run[read_index].read().first_cycle() - 1); tmp_by_tile[id] = 0; diff --git a/interop/logic/summary/error_summary.h b/interop/logic/summary/error_summary.h index 1ee18bf3f..dacdb948c 100644 --- a/interop/logic/summary/error_summary.h +++ b/interop/logic/summary/error_summary.h @@ -8,6 +8,7 @@ #pragma once #include +#include "interop/util/map.h" #include "interop/util/exception.h" #include "interop/util/length_of.h" #include "interop/model/model_exceptions.h" @@ -47,7 +48,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary typedef std::vector cycle_vector_t; typedef std::pair key_t; typedef std::pair value_t; - typedef std::map error_tile_t; + typedef INTEROP_ORDERED_MAP(key_t, value_t) error_tile_t; typedef std::vector error_by_read_tile_t; error_by_read_tile_t tmp(read_lane_cache.size()); cycle_vector_t max_error_cycle(read_lane_cache.size(), 0); @@ -66,7 +67,9 @@ namespace illumina { namespace interop { namespace logic { namespace summary INTEROP_ASSERTMSG(read_number < tmp.size(), read.number << " " << read.cycle_within_read << ", " << beg->cycle()); if (tmp[read_number].find(key) == tmp[read_number].end()) - tmp[read_number][key] = std::make_pair(0.0f, 0.0f); + { + tmp[read_number].insert(std::make_pair(key, std::make_pair(0.0f, 0.0f))); + } tmp[read_number][key].first += beg->error_rate(); tmp[read_number][key].second += 1; } diff --git a/interop/logic/summary/quality_summary.h b/interop/logic/summary/quality_summary.h index 8cc3d48d2..e36ae000c 100644 --- a/interop/logic/summary/quality_summary.h +++ b/interop/logic/summary/quality_summary.h @@ -80,7 +80,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary const size_t surface=0) { INTEROP_ASSERT(read_number < m_read_lane_cache.size()); - const size_t lane_surface_index = lane*m_surface_count+surface; + const size_t lane_surface_index = index_of(lane, surface); INTEROP_ASSERT(lane_surface_index < m_read_lane_cache[read_number].size()); m_read_lane_cache[read_number][lane_surface_index].above_qval+=metric.q30(); m_read_lane_cache[read_number][lane_surface_index].total += metric.total(); diff --git a/interop/logic/summary/run_summary.h b/interop/logic/summary/run_summary.h index 8ae37b271..83cbc4a5c 100644 --- a/interop/logic/summary/run_summary.h +++ b/interop/logic/summary/run_summary.h @@ -23,10 +23,12 @@ namespace illumina { namespace interop { namespace logic { namespace summary * @param metrics source collection of all metrics * @param summary destination run summary * @param skip_median skip the median calculation + * @param trim flag indicating whether to trim the summary model (default: true) */ void summarize_run_metrics(model::metrics::run_metrics& metrics, model::summary::run_summary& summary, - const bool skip_median=false) + const bool skip_median=false, + const bool trim=true) throw( model::index_out_of_bounds_exception, model::invalid_channel_exception ); diff --git a/interop/logic/summary/tile_summary.h b/interop/logic/summary/tile_summary.h index de5e6cee5..c65c6658f 100644 --- a/interop/logic/summary/tile_summary.h +++ b/interop/logic/summary/tile_summary.h @@ -23,12 +23,12 @@ namespace illumina { namespace interop { namespace logic { namespace summary * @param skip_median skip the median calculation */ template - void update_tile_summary_from_cache(std::vector &tile_data, - model::summary::stat_summary &stat_summary, + void update_tile_summary_from_cache(std::vector& tile_data, + model::summary::stat_summary& stat_summary, const bool skip_median) { model::summary::metric_stat stat; - summarize(tile_data.begin(), + nan_summarize(tile_data.begin(), tile_data.end(), stat, util::op::const_member_function(&model::metrics::tile_metric::cluster_density), @@ -36,7 +36,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary skip_median); stat_summary.density(stat); stat.clear(); - summarize(tile_data.begin(), + nan_summarize(tile_data.begin(), tile_data.end(), stat, util::op::const_member_function(&model::metrics::tile_metric::cluster_density_pf), @@ -44,7 +44,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary skip_median); stat_summary.density_pf(stat); stat.clear(); - summarize(tile_data.begin(), + nan_summarize(tile_data.begin(), tile_data.end(), stat, util::op::const_member_function(&model::metrics::tile_metric::cluster_count), @@ -52,7 +52,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary skip_median); stat_summary.cluster_count(stat); stat.clear(); - summarize(tile_data.begin(), + nan_summarize(tile_data.begin(), tile_data.end(), stat, util::op::const_member_function(&model::metrics::tile_metric::cluster_count_pf), @@ -60,7 +60,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary skip_median); stat_summary.cluster_count_pf(stat); stat.clear(); - summarize(tile_data.begin(), + nan_summarize(tile_data.begin(), tile_data.end(), stat, util::op::const_member_function(&model::metrics::tile_metric::percent_pf), @@ -78,7 +78,6 @@ namespace illumina { namespace interop { namespace logic { namespace summary util::op::const_member_function( &model::metrics::tile_metric::cluster_count_pf))); } - /** Update the stat summary with cached read metrics * * @param read_data_cache read metric cache @@ -86,9 +85,9 @@ namespace illumina { namespace interop { namespace logic { namespace summary * @param skip_median skip the median calculation * @return number of non-NaN aligned entries */ - inline size_t update_read_summary(summary_by_lane_read::vector_t &read_data_cache, - model::summary::stat_summary &stat_summary, - const bool skip_median) + inline size_t update_read_summary(summary_by_lane_read::vector_t& read_data_cache, + model::summary::stat_summary& stat_summary, + const bool skip_median) { model::summary::metric_stat stat; const size_t non_nan = nan_summarize(read_data_cache.begin(), @@ -118,7 +117,6 @@ namespace illumina { namespace interop { namespace logic { namespace summary stat_summary.phasing(stat); return non_nan; } - /** Summarize a collection tile metrics * * @sa model::summary::lane_summary::density @@ -146,8 +144,8 @@ namespace illumina { namespace interop { namespace logic { namespace summary I end, const constants::tile_naming_method naming_method, model::summary::run_summary &run, - const bool skip_median = false) - throw(model::index_out_of_bounds_exception) + const bool skip_median=false) + throw(model::index_out_of_bounds_exception) { typedef typename model::metrics::tile_metric::read_metric_vector read_metric_vector_t; typedef typename read_metric_vector_t::const_iterator const_read_metric_iterator; @@ -161,7 +159,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary tile_by_lane_vector_t tile_data_by_lane(run.lane_count()); reserve(tile_data_by_lane.begin(), tile_data_by_lane.end(), n); - tile_by_lane_vector_t tile_data_by_lane_surface(run.lane_count() * surface_count); + tile_by_lane_vector_t tile_data_by_lane_surface(run.lane_count()*surface_count); reserve(tile_data_by_lane_surface.begin(), tile_data_by_lane_surface.end(), n); summary_by_lane_read read_data_by_lane_read(run, n); @@ -180,13 +178,15 @@ namespace illumina { namespace interop { namespace logic { namespace summary { const size_t read = rb->read() - 1; if (read >= read_data_by_lane_read.read_count()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Read exceeds read count in RunInfo.xml"); - read_data_by_lane_read(rb->read() - 1, beg->lane() - 1).push_back(*rb); - if (surface_count < 2) continue; - read_data_by_surface_lane_read(rb->read() - 1, beg->lane() - 1, surface - 1).push_back(*rb); + INTEROP_THROW(model::index_out_of_bounds_exception, + "Read exceeds read count in RunInfo.xml: " + << read << " >= " << read_data_by_lane_read.read_count()); + read_data_by_lane_read(read, lane).push_back(*rb); + if(surface_count < 2) continue; + read_data_by_surface_lane_read(read, lane, surface-1).push_back(*rb); } - if (surface_count < 2) continue; - const size_t index = (beg->lane() - 1) * surface_count + (surface - 1); + if(surface_count < 2) continue; + const size_t index = lane*surface_count+(surface-1); tile_data_by_lane_surface[index].push_back(*beg);// TODO: make more efficient by copying only tile data } @@ -212,8 +212,8 @@ namespace illumina { namespace interop { namespace logic { namespace summary run[read][lane].reads(run[0][lane].reads()); run[read][lane].reads_pf(run[0][lane].reads_pf()); } - if (surface_count < 2) continue; - for (size_t surface = 0; surface < surface_count; ++surface) + if(surface_count < 2) continue; + for(size_t surface=0;surface &filled) { @@ -96,11 +98,26 @@ namespace illumina { namespace interop { namespace logic { namespace table * set_filled_metric_laneVoid(metric, read, q20_idx, q30_idx, naming_method, filled); */ # define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) \ - set_filled_metric_##Method##Param(metric, read, static_cast(q20_idx), static_cast(q30_idx), naming_method, filled); + set_filled_metric_##Method##Param(metric, read, static_cast(q20_idx), static_cast(q30_idx), cluster_count_k, naming_method, filled); INTEROP_IMAGING_COLUMN_TYPES # undef INTEROP_TUPLE7 // Reuse this for another conversion } + /** Get the maximum number of digits to round + * + * @return maximum number of rounding digits + */ + static uint32_t max_digits() + { +# define INTEROP_TUPLE7(Ignored0, Ignored1, Ignored2, Ignored3, Ignored4, Ignored5, Round) Round, + uint32_t digits[] = { INTEROP_IMAGING_COLUMN_TYPES 0}; +# undef INTEROP_TUPLE7 + uint32_t max_digit = 0; + for(size_t i=0;i& filled)\ {\ if(is_valid(call_adapter(metric, Param, &model:: Metric::Method))) filled[model::table::Id##Column] = true;\ - (void)Q20;(void)Q30;(void)NamingConvention;(void)Read;\ + (void)Q20;(void)Q30;(void)NamingConvention;(void)Read;(void)ClusterCountK;\ }\ static void set_filled_metric_##Method##Param(const model::metric_base::empty_metric&,\ const size_t,\ const uint_t,\ const uint_t,\ + const float,\ const constants::tile_naming_method,\ std::vector&){} INTEROP_IMAGING_COLUMN_TYPES @@ -179,9 +198,9 @@ namespace illumina { namespace interop { namespace logic { namespace table * @return true */ template - static bool is_valid(const T) + static bool is_valid(const T val) { - return true; + return val < std::numeric_limits::max(); } /** Test if a metric type is valid diff --git a/interop/logic/table/create_imaging_table.h b/interop/logic/table/create_imaging_table.h index ed255a501..4cac07ccb 100644 --- a/interop/logic/table/create_imaging_table.h +++ b/interop/logic/table/create_imaging_table.h @@ -10,9 +10,14 @@ #include "interop/model/run_metrics.h" #include "interop/model/table/imaging_column.h" #include "interop/model/table/imaging_table.h" +#include "interop/logic/table/table_util.h" namespace illumina { namespace interop { namespace logic { namespace table { + // TODO: Make unordered? - need to sort later + // Workaround for SWIG not understanding the macro + /** Define a row offset map */ + typedef std::map row_offset_map_t; /** Populate the imaging table with all the metrics in the run * * @param metrics collection of all run metrics @@ -23,7 +28,7 @@ namespace illumina { namespace interop { namespace logic { namespace table */ void populate_imaging_table_data(const model::metrics::run_metrics& metrics, const std::vector& columns, - const std::map& row_offset, + const row_offset_map_t& row_offset, float* data_beg, const size_t n); /** Count the number of rows in the imaging table and setup an ordering * @@ -31,7 +36,7 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param row_offset ordering for the rows */ void count_table_rows(const model::metrics::run_metrics& metrics, - std::map& row_offset); + row_offset_map_t& row_offset); /** Count the total number of columns for the data table * * @param columns vector of table column descriptions @@ -43,7 +48,7 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param metrics source run metrics * @param table destination imaging table */ - void create_imaging_table(const model::metrics::run_metrics& metrics, model::table::imaging_table& table) + void create_imaging_table(model::metrics::run_metrics& metrics, model::table::imaging_table& table) throw(model::invalid_column_type, model::index_out_of_bounds_exception); /** List the required on demand metrics diff --git a/interop/logic/table/create_imaging_table_columns.h b/interop/logic/table/create_imaging_table_columns.h index 34c5cc6a6..4207fcb54 100644 --- a/interop/logic/table/create_imaging_table_columns.h +++ b/interop/logic/table/create_imaging_table_columns.h @@ -14,6 +14,11 @@ namespace illumina { namespace interop { namespace logic { namespace table { + /** Get the maximum number of digits to round + * + * @return maximum number of rounding digits + */ + ::uint32_t max_digits(); /** Get the number of digits to round a column * * @param index imaging table column id @@ -29,7 +34,9 @@ namespace illumina { namespace interop { namespace logic { namespace table void create_imaging_table_columns(const std::vector& channels, const std::vector& filled, std::vector< model::table::imaging_column >& columns) - throw(model::invalid_column_type, model::index_out_of_bounds_exception); + throw(model::invalid_column_type, + model::index_out_of_bounds_exception, + model::invalid_channel_exception); /** Populate the value offsets from the column headers * * @param columns column headers @@ -41,9 +48,11 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param metrics source collection of InterOp metrics from the run * @param columns destination vector of column descriptors */ - void create_imaging_table_columns(const model::metrics::run_metrics& metrics, + void create_imaging_table_columns(model::metrics::run_metrics& metrics, std::vector< model::table::imaging_column >& columns) - throw(model::invalid_column_type, model::index_out_of_bounds_exception); + throw(model::invalid_column_type, + model::index_out_of_bounds_exception, + model::invalid_channel_exception); }}}} diff --git a/interop/logic/table/table_populator.h b/interop/logic/table/table_populator.h index d37f55e9d..06a937f62 100644 --- a/interop/logic/table/table_populator.h +++ b/interop/logic/table/table_populator.h @@ -40,6 +40,7 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param read read number, cycle within read * @param q20_idx index of the q20 value * @param q30_idx index of the q30 value + * @param cluster_count_k cluster count in kilobases * @param naming_method tile naming method enum * @param columns vector of table columns * @param data_it iterator to current row of table data @@ -50,6 +51,7 @@ namespace illumina { namespace interop { namespace logic { namespace table const summary::read_cycle &read, const size_t q20_idx, const size_t q30_idx, + const float cluster_count_k, const constants::tile_naming_method naming_method, const std::vector &columns, OutputIterator data_it, OutputIterator data_end) @@ -58,6 +60,7 @@ namespace illumina { namespace interop { namespace logic { namespace table read.number, q20_idx, q30_idx, + cluster_count_k, naming_method, columns, data_it, @@ -66,6 +69,7 @@ namespace illumina { namespace interop { namespace logic { namespace table read.number, q20_idx, q30_idx, + cluster_count_k, naming_method, columns, data_it, @@ -81,6 +85,7 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param read read number, cycle within read * @param q20_idx index of the q20 value * @param q30_idx index of the q30 value + * @param cluster_count_k cluster count in kilobases * @param naming_method tile naming method enum * @param columns vector of table columns * @param data_it iterator to current row of table data @@ -91,6 +96,7 @@ namespace illumina { namespace interop { namespace logic { namespace table const size_t read, const size_t q20_idx, const size_t q30_idx, + const float cluster_count_k, const constants::tile_naming_method naming_method, const std::vector &columns, OutputIterator data_it, OutputIterator data_end) @@ -104,7 +110,7 @@ namespace illumina { namespace interop { namespace logic { namespace table * update_laneVoid(metric, q20_idx, q30_idx, naming_convention); */ # define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) \ - populate_##Method##Param(metric, read, static_cast(q20_idx), static_cast(q30_idx), naming_method, columns, data_it, data_end); + populate_##Method##Param(metric, read, static_cast(q20_idx), static_cast(q30_idx), cluster_count_k, naming_method, columns, data_it, data_end); INTEROP_IMAGING_COLUMN_TYPES # undef INTEROP_TUPLE7 // Reuse this for another conversion } @@ -128,20 +134,23 @@ namespace illumina { namespace interop { namespace logic { namespace table const size_t Read,\ const uint_t Q20,\ const uint_t Q30,\ + const float ClusterCountK,\ const constants::tile_naming_method NamingConvention,\ const std::vector& columns,\ OutputIterator data_it, OutputIterator data_end)\ {\ + INTEROP_ASSERT( model::table:: Id##Column < columns.size() ); \ const size_t index = columns[model::table:: Id##Column];\ - if(index == std::numeric_limits::max()) return; /*Missing column */ \ + if(!is_valid(index)) return; /*Missing column */ \ copy_to(data_it+index, data_end, call_adapter(metric, Param, &model:: Metric::Method), Round);\ - (void)Q20;(void)Q30;(void)NamingConvention;(void)Read;\ + (void)Q20;(void)Q30;(void)NamingConvention;(void)Read;(void)ClusterCountK;\ }\ template\ static void populate_##Method##Param(const MetricType&,\ const size_t,\ const uint_t,\ const uint_t,\ + const float,\ const constants::tile_naming_method,\ const std::vector&,\ OutputIterator,OutputIterator){} @@ -160,7 +169,8 @@ namespace illumina { namespace interop { namespace logic { namespace table static void assign(T &destination, const U source, const size_t num_digits = 0) { //INTEROP_ASSERTMSG(std::isnan(destination), destination); - destination = static_cast(roundto(source, num_digits)); + if(is_valid(source)) + destination = static_cast(roundto(source, num_digits)); } private: @@ -229,9 +239,9 @@ namespace illumina { namespace interop { namespace logic { namespace table * @return true */ template - static bool is_valid(const T) + static bool is_valid(const T val) { - return true; + return val < std::numeric_limits::max(); } /** Test if a metric type is valid diff --git a/interop/logic/utils/channel.h b/interop/logic/utils/channel.h index 42192848c..543ffbf3a 100644 --- a/interop/logic/utils/channel.h +++ b/interop/logic/utils/channel.h @@ -95,7 +95,7 @@ namespace illumina { namespace interop { namespace logic { namespace utils std::swap(expected[0], expected[1]); return expected; } - INTEROP_THROW( model::invalid_channel_exception, "Invalid channel names"); + INTEROP_THROW( model::invalid_channel_exception, "Invalid channel names: " << norm); } /** Expected channel order @@ -138,6 +138,8 @@ namespace illumina { namespace interop { namespace logic { namespace utils normed.reserve(channels.size()); normalize(channels.begin(), channels.end(), std::back_inserter(normed)); std::vector expected = expected_order(normed); + map.resize(normed.size()); + INTEROP_ASSERTMSG(expected.size() == normed.size(), expected.size() << " == " << normed.size()); for(size_t i=0;i + class enumeration_string_mapping< plot_types > + { + typedef std::pair name_type_pair_t; + public: + /** Pass an array of string, enum pairs and its length to the given function + * + * @param func pointer to a function that takes an array of string/enum pairs as a parameter + * @return the value returned by the given function (in the case of `parse` and `to_string` the return value is a singleton) + */ + template + static R setup(F func) + { + static const name_type_pair_t name_types[] = {INTEROP_ENUM_PLOT_TYPES}; + return func(name_types, util::length_of(name_types)); + } + }; + + /** Specialization that maps metric base types to a string + */ + template<> + class enumeration_string_mapping< metric_base_type > + { + typedef std::pair name_type_pair_t; + public: + /** Pass an array of string, enum pairs and its length to the given function + * + * @param func pointer to a function that takes an array of string/enum pairs as a parameter + * @return the value returned by the given function (in the case of `parse` and `to_string` the return value is a singleton) + */ + template + static R setup(F func) + { + static const name_type_pair_t name_types[] = {INTEROP_ENUM_METRIC_BASE_TYPES}; + return func(name_types, util::length_of(name_types)); + } + }; + }}} #undef INTEROP_TUPLE1 diff --git a/interop/logic/utils/metric_type_ext.h b/interop/logic/utils/metric_type_ext.h index 5b74307ec..f1f1b123e 100644 --- a/interop/logic/utils/metric_type_ext.h +++ b/interop/logic/utils/metric_type_ext.h @@ -8,6 +8,7 @@ #pragma once #include "interop/util/constant_mapping.h" #include "interop/constants/enums.h" +#include "interop/constants/enum_description.h" #include "interop/logic/utils/enums.h" namespace illumina { namespace interop { namespace logic { namespace utils @@ -72,6 +73,7 @@ namespace illumina { namespace interop { namespace logic { namespace utils # undef INTEROP_TUPLE4 return util::constant_mapping_get(name_types, type, UnknownMetricFeature); } + /** Test if metric type is indexed by DNA base * * @param type metric type diff --git a/interop/logic/utils/metrics_to_load.h b/interop/logic/utils/metrics_to_load.h index e88f78389..8de176552 100644 --- a/interop/logic/utils/metrics_to_load.h +++ b/interop/logic/utils/metrics_to_load.h @@ -8,6 +8,7 @@ #pragma once #include #include "interop/constants/enums.h" +#include "interop/constants/enum_description.h" #include "interop/model/model_exceptions.h" namespace illumina { namespace interop { namespace logic { namespace utils diff --git a/interop/model/metric_base/base_cycle_metric.h b/interop/model/metric_base/base_cycle_metric.h index 3ee887e8f..aa658290c 100644 --- a/interop/model/metric_base/base_cycle_metric.h +++ b/interop/model/metric_base/base_cycle_metric.h @@ -96,6 +96,16 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { } public: + /** Set id + * + * @note This is only supported to enable easier unit testing, this should not be used. + * @param lane lane number + * @param tile tile number + */ + void set_base(const uint_t lane, const uint_t tile) + { + base_metric::set_base(lane, tile); + } /** Set id * * @param lane lane number @@ -133,6 +143,14 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { return create_id(lane(), tile(), m_cycle); } + /** Unique id created from the lane, tile and cycle + * + * @return unique id + */ + id_t cycle_hash() const + { + return create_id(lane(), tile(), m_cycle); + } /** Get the cycle from the unique lane/tile/cycle id * * @param id unique lane/tile/cycle id @@ -140,8 +158,20 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t cycle_from_id(const id_t id) { - // Mask tile hash (lane + tile) - return id & ~((~static_cast(0)) << TILE_BIT_SHIFT); + // 1. Shift off lane and tile bits + // 2. Shift back, while shifting off reserved bits + return (id << (LANE_BIT_COUNT+TILE_BIT_COUNT)) >> (LANE_BIT_COUNT+TILE_BIT_COUNT+RESERVED_BIT_COUNT); + } + /** Get the reserved number from the unique lane/tile/cycle id + * + * @param id unique lane/tile/cycle id + * @return reserved number + */ + static id_t reserved_from_id(const id_t id) + { + // 1. Shift off lane, tile and cycle bits + // 2. Shift back + return (id << (LANE_BIT_COUNT+TILE_BIT_COUNT+CYCLE_BIT_COUNT)) >> (LANE_BIT_COUNT+TILE_BIT_COUNT+CYCLE_BIT_COUNT); } /** Create unique id from the lane, tile and cycle @@ -153,7 +183,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t create_id(const id_t lane, const id_t tile, const id_t cycle) { - return base_metric::create_id(lane, tile) | cycle; + return base_metric::create_id(lane, tile) | (cycle << CYCLE_BIT_SHIFT); } private: diff --git a/interop/model/metric_base/base_metric.h b/interop/model/metric_base/base_metric.h index 052f1b0bf..0fd6f2bc0 100644 --- a/interop/model/metric_base/base_metric.h +++ b/interop/model/metric_base/base_metric.h @@ -94,18 +94,25 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { TYPE=constants::UnknownMetricType, //Compress Lane, Tile and cycle into a 64-bit Integer - // Cycle: Bits 0-32 + // Bit order: LTCR or Lane, Tile, Cycle/Read, Reserved + // Reserved: Bits 0-16 + // Read/Cycle: Bits 16-32 // Tile: Bits 32-58 // Lane: Bits 58-64 LANE_BIT_COUNT = 6, // Supports up to 63 lanes TILE_BIT_COUNT = 26, // Support 7 digit tile (up to 67108864) - CYCLE_BIT_COUNT = 32, // Support up to 4294967296 cycles - LANE_BIT_SHIFT = CYCLE_BIT_COUNT+TILE_BIT_COUNT, // Supports up to 63 lanes - TILE_BIT_SHIFT =CYCLE_BIT_COUNT, + CYCLE_BIT_COUNT = 16, // Support up to 65535 cycles + READ_BIT_COUNT = 16, // Support up to 65535 reads + RESERVED_BIT_COUNT = 16, // Support up to 65535 values + READ_BIT_SHIFT = RESERVED_BIT_COUNT, + CYCLE_BIT_SHIFT = RESERVED_BIT_COUNT, + EVENT_BIT_SHIFT = 0, + TILE_BIT_SHIFT = CYCLE_BIT_SHIFT+CYCLE_BIT_COUNT, + LANE_BIT_SHIFT = TILE_BIT_SHIFT+TILE_BIT_COUNT, // Supports up to 63 lanes /** Base for records written out once for each tile */ BASE_TYPE = constants::BaseTileType, - /** Tells the reader to exclude any records that have 0 for tile */ - CHECK_TILE_ID = 1 + /** Latest version is sentinel*/ + LATEST_VERSION=0 }; public: /** Constructor @@ -191,7 +198,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t tile_hash_from_id(const id_t id) { - // 1. Remove cycle information + // 1. Remove cycle and reserved information return (id >> TILE_BIT_SHIFT) << TILE_BIT_SHIFT; } /** Get the tile hash from the unique lane/tile/cycle id @@ -201,9 +208,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t tile_from_id(const id_t id) { - // 1. Mask out lane - // 2. Shift tile id - return (id & ~((~static_cast(0)) << LANE_BIT_SHIFT)) >> TILE_BIT_SHIFT; + // 1. Shift off lane + // 2. Shift tile id into proper position (removing cycle and reserved) + return (id << (LANE_BIT_COUNT) >> (LANE_BIT_COUNT+CYCLE_BIT_COUNT+RESERVED_BIT_COUNT)); } /** Lane number diff --git a/interop/model/metric_base/base_read_metric.h b/interop/model/metric_base/base_read_metric.h index bb9019af0..f2fd10fd0 100644 --- a/interop/model/metric_base/base_read_metric.h +++ b/interop/model/metric_base/base_read_metric.h @@ -79,7 +79,16 @@ namespace illumina { namespace interop { namespace model { namespace metric_base base_metric(lane, tile), m_read(read) { } public: - + /** Set id + * + * @note This is only supported to enable easier unit testing, this should not be used. + * @param lane lane number + * @param tile tile number + */ + void set_base(const uint_t lane, const uint_t tile) + { + base_metric::set_base(lane, tile); + } /** * Sets ID given lane-tile-read as opposed to the BaseReadMetric * @@ -97,13 +106,12 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param base layout base */ template - void set_base(const BaseReadMetric &base) { + void set_base(const BaseReadMetric &base) + { base_metric::set_base(base); m_read = base.read; } - - /** Read number * * @return read number @@ -128,7 +136,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t create_id(const id_t lane, const id_t tile, const id_t read) { - return base_metric::create_id(lane, tile) | read; + return base_metric::create_id(lane, tile) | (read << READ_BIT_SHIFT); } /** Get the read from the unique lane/tile/read id * @@ -137,8 +145,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t read_from_id(const id_t id) { - // Mask tile hash (lane + tile) - return id & ~((~static_cast(0)) << TILE_BIT_SHIFT); + // 1. Shift off lane and tile bits + // 2. Shift back, while shifting off reserved bits + return (id << (LANE_BIT_COUNT+TILE_BIT_COUNT)) >> (LANE_BIT_COUNT+TILE_BIT_COUNT+RESERVED_BIT_COUNT); } private: diff --git a/interop/model/metric_base/metric_set.h b/interop/model/metric_base/metric_set.h index aab533940..ea88d3a59 100644 --- a/interop/model/metric_base/metric_set.h +++ b/interop/model/metric_base/metric_set.h @@ -7,12 +7,12 @@ */ #pragma once -#include #include #include #include #include #include +#include "interop/util/map.h" #include "interop/util/exception.h" #include "interop/model/metric_base/base_cycle_metric.h" #include "interop/model/metric_base/base_read_metric.h" @@ -58,7 +58,13 @@ namespace illumina { namespace interop { namespace model { namespace metric_base /** Define the size type */ typedef typename metric_array_t::size_type size_type; /** Define a set of ids */ - typedef std::set id_set_t; + typedef std::set id_set_t; // TODO: Do the same for set + /** Define offset map */ +#if defined(__cplusplus) && __cplusplus < 201103L // Workaround for SWIG not understanding the macro + typedef std::map offset_map_t; +#else + typedef std::unordered_map offset_map_t; +#endif public: /** Const metric iterator */ @@ -67,10 +73,10 @@ namespace illumina { namespace interop { namespace model { namespace metric_base typedef typename metric_array_t::iterator iterator; enum { /** Group type enum */ - TYPE=T::TYPE + TYPE=T::TYPE, + /** Latest version of the format */ + LATEST_VERSION=T::LATEST_VERSION }; - private: - typedef std::map id_map_t; public: /** Constructor @@ -101,7 +107,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base m_version(version), m_data_source_exists(false) { - rebuild_index(); + rebuild_index(true); } public: @@ -160,39 +166,34 @@ namespace illumina { namespace interop { namespace model { namespace metric_base public: /** Rebuild the index map and update the cycle state */ - void rebuild_index() + void rebuild_index(const bool update_ids=false) { size_t offset = 0; - for (const_iterator b = m_data.begin(), e = m_data.end(); b != e; ++b) + for (const_iterator b = begin(), e = end(); b != e; ++b) { - m_id_map[b->id()] = offset; - ++offset; + if(update_ids) + { + m_id_map[b->id()] = offset; + ++offset; + } T::header_type::update_max_cycle(*b); } } - /** Swap the contents of the internal vector with an external vector - * - * @param dest source or destination vector - */ - void swap(metric_array_t& dest) - { - m_data.swap(dest); - } /** Resert the number of places in the metric vector * * @param n expected number of elements */ - void reserve(const size_t n) + void resize(const size_t n) { - m_data.reserve(n); + m_data.resize(n, metric_type(*this)); } - /** Resert the number of places in the metric vector + /** Trim the set to the proper number of metrics * - * @param n expected number of elements + * @param map proper number of metrics */ - void resize(const size_t n) + void trim(const size_t n) { - m_data.resize(n, metric_type(*this)); + m_data.resize(n); } /** Find index of metric given the id. If not found, return number of metrics * @@ -213,7 +214,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ size_t find(const id_t id) const { - typename id_map_t::const_iterator it = m_id_map.find(id); + typename offset_map_t::const_iterator it = m_id_map.find(id); if (it == m_id_map.end()) return size(); return it->second; } @@ -256,7 +257,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { INTEROP_ASSERT(id != 0); // TODO: remove the following - m_id_map[id] = m_data.size(); + m_id_map[id] = size(); T::header_type::update_max_cycle(metric); m_data.push_back(metric); @@ -271,8 +272,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param cycle cycle * @return metric */ - const metric_type &get_metric(uint_t lane, uint_t tile, - uint_t cycle = 0) const throw(model::index_out_of_bounds_exception) + const metric_type &get_metric(const uint_t lane, const uint_t tile, + const uint_t cycle = 0) const throw(model::index_out_of_bounds_exception) { try { @@ -286,7 +287,6 @@ namespace illumina { namespace interop { namespace model { namespace metric_base " lane: " << (lane) << " tile: " << (tile) << " cycle: " << (cycle)); - } } @@ -297,9 +297,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param key unique id built from lane, tile and cycle (if available) * @return metric */ - const metric_type &get_metric(id_t key) const throw(model::index_out_of_bounds_exception) + const metric_type &get_metric(const id_t key) const throw(model::index_out_of_bounds_exception) { - typename std::map::const_iterator it = m_id_map.find(key); + typename offset_map_t::const_iterator it = m_id_map.find(key); if (it == m_id_map.end()) INTEROP_THROW( index_out_of_bounds_exception, "No tile available: key: " << key << " map: " << m_id_map.size() << " == data: " << @@ -313,7 +313,18 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param n index * @return metric */ - metric_type &at(size_t n) throw(index_out_of_bounds_exception) + metric_type &at(const size_t n) throw(index_out_of_bounds_exception) + { + if(n >= m_data.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index out of bounds"); + return m_data.at(n); + } + + /** Get a metric at the given index + * + * @param n index + * @return metric + */ + const metric_type &at(const size_t n)const throw(index_out_of_bounds_exception) { if(n >= m_data.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index out of bounds"); return m_data.at(n); @@ -340,7 +351,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base key_vector keys() const { key_vector ids; - std::transform(m_data.begin(), m_data.end(), std::back_inserter(ids), to_id); + std::transform(begin(), end(), std::back_inserter(ids), to_id); return ids; } @@ -350,8 +361,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ id_vector lanes() const { - std::set id_set; - std::transform(m_data.begin(), m_data.end(), std::inserter(id_set, id_set.begin()), to_lane); + id_set_t id_set; + std::transform(begin(), end(), std::inserter(id_set, id_set.begin()), to_lane); return id_vector(id_set.begin(), id_set.end()); } @@ -371,7 +382,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base size_t max_lane() const { size_t lane_max = 0; - for (const_iterator b = m_data.begin(), e = m_data.end(); b != e; ++b) + for (const_iterator b = begin(); b != end(); ++b) lane_max = std::max(lane_max, static_cast(b->lane())); return lane_max; } @@ -383,17 +394,27 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ id_vector tile_numbers_for_lane(const uint_t lane) const { - std::set tile_number_set; - transform_if(m_data.begin(), - m_data.end(), + id_set_t tile_number_set; + populate_tile_numbers_for_lane(tile_number_set, lane); + id_vector tile_numbers(tile_number_set.begin(), tile_number_set.end()); + return tile_numbers; + } + /** Get a list of all available tile numbers for the specified lane + * + * @note this does not clear the set! + * + * @param tile_number_set destination set to ensure unique tile numbers + * @param lane lane number + */ + void populate_tile_numbers_for_lane(id_set_t& tile_number_set, const uint_t lane) const + { + transform_if(begin(), + end(), std::inserter(tile_number_set, tile_number_set.begin()), lane_equals(lane), to_tile); - id_vector tile_numbers(tile_number_set.begin(), tile_number_set.end()); - return tile_numbers; } - - /** Get a list of all available tile numbers for the specified lane/surface + /** Get a list of all available tile numbers for the specified lane * * @note this does not clear the set! * @@ -402,13 +423,13 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param surface surface number * @param naming_convention tile naming convetion enum */ - void tile_numbers_for_lane_surface(id_set_t& tile_number_set, - const uint_t lane, - const uint_t surface, - const constants::tile_naming_method naming_convention) const + void populate_tile_numbers_for_lane_surface(id_set_t& tile_number_set, + const uint_t lane, + const uint_t surface, + const constants::tile_naming_method naming_convention) const { - transform_if(m_data.begin(), - m_data.end(), + transform_if(begin(), + end(), std::inserter(tile_number_set, tile_number_set.begin()), lane_surface_equals(lane, surface, naming_convention), to_tile); @@ -420,9 +441,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ id_vector tile_numbers() const { - std::set tile_number_set; - transform(m_data.begin(), - m_data.end(), + id_set_t tile_number_set; + transform(begin(), + end(), std::inserter(tile_number_set, tile_number_set.begin()), to_tile); id_vector tile_numbers(tile_number_set.begin(), tile_number_set.end()); @@ -437,24 +458,45 @@ namespace illumina { namespace interop { namespace model { namespace metric_base metric_array_t metrics_for_lane(const uint_t lane) const { metric_array_t lane_metrics; + metrics_for_lane(lane_metrics, lane); + return lane_metrics; + } + + /** Get a list of all available metrics for the specified lane + * + * @param lane_metrics destination metric array + * @param lane lane number + */ + void metrics_for_lane(metric_array_t& lane_metrics, const uint_t lane) const + { lane_metrics.reserve(size()); - copy_if(m_data.begin(), - m_data.end(), + copy_if(begin(), + end(), std::back_inserter(lane_metrics), lane_equals(lane)); metric_array_t(lane_metrics).swap(lane_metrics); // Shrink to fit - return lane_metrics; + } + + /** Get a list of all cycles listed in the metric set + * + * @note Returns empty array for metrics that do not have a cycle identifier + * @return vector of cycle numbers + */ + id_vector cycles() const + { + id_set_t cycle_set; + cycles(cycle_set); + return id_vector(cycle_set.begin(), cycle_set.end()); } /** Get a list of all available metrics for the specified cycle * * @note Returns empty array for Tile Metrics, which does not contain a cycle identifier * @param cycle cycle number - * @return vector of tile numbers + * @return vector of metrics that map to the given cycle */ metric_array_t metrics_for_cycle(const uint_t cycle) const { - typedef typename metric_type::base_t base_t; return metrics_for_cycle(cycle, base_t::null()); } @@ -556,21 +598,20 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ metric_type &get_metric_ref(id_t key) throw(model::index_out_of_bounds_exception) { - typename std::map::const_iterator it = m_id_map.find(key); + typename offset_map_t::const_iterator it = m_id_map.find(key); if (it == m_id_map.end()) INTEROP_THROW( index_out_of_bounds_exception, "No tile available: key: " << (key) << " map: " << (m_id_map.size()) << " == data: " << - (m_data.size())); - INTEROP_ASSERT(it->second < m_data.size()); + (size())); + INTEROP_ASSERT(it->second < size()); return m_data[it->second]; } - /** Get the current id offset map * * @return id offset map */ - const id_map_t& offset_map()const + offset_map_t& offset_map() { return m_id_map; } @@ -580,8 +621,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { metric_array_t cycle_metrics; cycle_metrics.reserve(size()); - copy_if(m_data.begin(), - m_data.end(), + copy_if(begin(), + end(), std::back_inserter(cycle_metrics), cycle_equals(cycle)); metric_array_t(cycle_metrics).swap(cycle_metrics); // Shrink to fit @@ -609,6 +650,27 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { return metric.tile(); } + + void cycles(id_set_t& cycles_set) const + { + cycles(cycles_set, base_t::null()); + } + + void cycles(id_set_t& cycles_set, const constants::base_cycle_t*) const + { + std::transform(begin(), end(), std::inserter(cycles_set, cycles_set.begin()), to_cycle); + } + + void cycles(id_set_t&, const void *) const + { + } + + static uint_t to_cycle(const metric_type &metric) + { + //TODO: ensure that this isn't called from a non-base_cycle metric set? + return metric.cycle(); + } + template static OIterator transform(I beg, I end, OIterator it, Operation op) { @@ -646,15 +708,14 @@ namespace illumina { namespace interop { namespace model { namespace metric_base struct lane_equals { - lane_equals(uint_t lane) : m_lane(lane) + lane_equals(const uint_t lane) : m_lane(lane) { } bool operator()(const metric_type &metric) const { return metric.lane() == m_lane; } - uint_t m_lane; + const uint_t m_lane; }; - struct lane_surface_equals { lane_surface_equals(const uint_t lane, @@ -697,7 +758,22 @@ namespace illumina { namespace interop { namespace model { namespace metric_base // TODO: remove the following /** Map unique identifiers to the index of the metric */ - id_map_t m_id_map; + offset_map_t m_id_map; + }; + + /** Get metric set for a given metric set */ + template + struct metric_set_helper + { + /** Define a metric set type */ + typedef metric_set metric_set_t; + }; + /** Get metric set for a given metric */ + template + struct metric_set_helper< metric_set > + { + /** Define a metric set type */ + typedef metric_set metric_set_t; }; }}}} #ifdef _MSC_VER diff --git a/interop/model/metrics/corrected_intensity_metric.h b/interop/model/metrics/corrected_intensity_metric.h index 7ded9c499..30243d35d 100644 --- a/interop/model/metrics/corrected_intensity_metric.h +++ b/interop/model/metrics/corrected_intensity_metric.h @@ -79,9 +79,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics corrected_intensity_metric() : metric_base::base_cycle_metric(0, 0, 0), m_average_cycle_intensity(0), - m_corrected_int_all(constants::NUM_OF_BASES, 0), - m_corrected_int_called(constants::NUM_OF_BASES, 0), - m_called_counts(constants::NUM_OF_BASES_AND_NC, 0), + m_corrected_int_all(constants::NUM_OF_BASES, std::numeric_limits::max()), + m_corrected_int_called(constants::NUM_OF_BASES, std::numeric_limits::max()), + m_called_counts(constants::NUM_OF_BASES_AND_NC, std::numeric_limits::max()), m_signal_to_noise(std::numeric_limits::quiet_NaN()) { } /** Constructor @@ -89,9 +89,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics corrected_intensity_metric(const header_type&) : metric_base::base_cycle_metric(0, 0, 0), m_average_cycle_intensity(0), - m_corrected_int_all(constants::NUM_OF_BASES, 0), - m_corrected_int_called(constants::NUM_OF_BASES, 0), - m_called_counts(constants::NUM_OF_BASES_AND_NC, 0), + m_corrected_int_all(constants::NUM_OF_BASES, std::numeric_limits::max()), + m_corrected_int_called(constants::NUM_OF_BASES, std::numeric_limits::max()), + m_called_counts(constants::NUM_OF_BASES_AND_NC, std::numeric_limits::max()), m_signal_to_noise(std::numeric_limits::quiet_NaN()) { } @@ -175,7 +175,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics const uint_array_t &called_counts) : metric_base::base_cycle_metric(lane, tile, cycle), m_average_cycle_intensity(std::numeric_limits::max()), - m_corrected_int_all(constants::NUM_OF_BASES, 0), + m_corrected_int_all(constants::NUM_OF_BASES, std::numeric_limits::max()), m_corrected_int_called(corrected_int_called), m_called_counts(called_counts), m_signal_to_noise(std::numeric_limits::quiet_NaN()) @@ -200,7 +200,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics const uint_pointer_t called_counts) : metric_base::base_cycle_metric(lane, tile, cycle), m_average_cycle_intensity(std::numeric_limits::max()), - m_corrected_int_all(constants::NUM_OF_BASES, 0), + m_corrected_int_all(constants::NUM_OF_BASES, std::numeric_limits::max()), m_corrected_int_called(corrected_int_called, corrected_int_called + constants::NUM_OF_BASES), m_called_counts(called_counts, called_counts + constants::NUM_OF_BASES_AND_NC), m_signal_to_noise(std::numeric_limits::quiet_NaN()) diff --git a/interop/model/metrics/extraction_metric.h b/interop/model/metrics/extraction_metric.h index 19c89d645..a34df1251 100644 --- a/interop/model/metrics/extraction_metric.h +++ b/interop/model/metrics/extraction_metric.h @@ -277,7 +277,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @return number of channels */ - size_t channel_count() const + size_t channel_count()const { return m_focus_scores.size(); } @@ -323,6 +323,15 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ const float_array_t &focusScores() const { return m_focus_scores; } + /** Set the current extraction time + * + * @param time current time + */ + void date_time(const ulong_t time) + { + m_date_time=time; + m_date_time_csharp = util::csharp_date_time::to_csharp(time); + } public: /** Get the prefix of the InterOp filename diff --git a/interop/model/metrics/index_metric.h b/interop/model/metrics/index_metric.h index 6e1829c1a..bbb37374e 100644 --- a/interop/model/metrics/index_metric.h +++ b/interop/model/metrics/index_metric.h @@ -41,7 +41,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics m_index_seq(""), m_sample_id(""), m_sample_proj(""), - m_count(0) + m_cluster_count(0) { } @@ -50,16 +50,16 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param index_seq index sequence * @param sample_id sample id * @param sample_proj sample project - * @param count number of index sequences + * @param cluster_count number of index sequences */ index_info(const std::string &index_seq, const std::string &sample_id, const std::string &sample_proj, - const size_t count) : + const ::uint64_t cluster_count) : m_index_seq(index_seq), m_sample_id(sample_id), m_sample_proj(sample_proj), - m_count(count) + m_cluster_count(cluster_count) { } @@ -97,8 +97,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @return number of clusters */ - size_t count() const - { return m_count; } + ::uint64_t cluster_count() const + { return m_cluster_count; } /** Test if the sequence is a dual index * @@ -145,7 +145,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics std::string m_index_seq; std::string m_sample_id; std::string m_sample_proj; - size_t m_count; + ::uint64_t m_cluster_count; template friend struct io::generic_layout; diff --git a/interop/model/metrics/q_by_lane_metric.h b/interop/model/metrics/q_by_lane_metric.h index 03a3ec61f..3f894d111 100644 --- a/interop/model/metrics/q_by_lane_metric.h +++ b/interop/model/metrics/q_by_lane_metric.h @@ -22,10 +22,11 @@ namespace illumina { namespace interop { namespace model { namespace metrics { public: enum { + /** Unique type code for metric */ TYPE = constants::QByLane, - /** Tells the reader to include any records that have 0 for tile */ - CHECK_TILE_ID = 0 + /** Latest version of the InterOp format */ + LATEST_VERSION = 6 }; /** Define the base type */ typedef constants::base_lane_t base_t; diff --git a/interop/model/metrics/q_metric.h b/interop/model/metrics/q_metric.h index 5180eb106..a9ae7b355 100644 --- a/interop/model/metrics/q_metric.h +++ b/interop/model/metrics/q_metric.h @@ -151,7 +151,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @return vector of q-score bins */ - const qscore_bin_vector_type &bins() const + const qscore_bin_vector_type &get_bins() const { return m_qscore_bins; } @@ -262,6 +262,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** Defines a vector of unsigned 32-bit ints */ typedef std::vector< ::uint32_t > uint32_vector; + /** Defines a vector of unsigned 32-bit ints (TODO: remove this def) + */ + typedef std::vector< ::uint32_t > uint_vector; /** Defines a vector of unsigned 64-bit ints */ typedef std::vector< ::uint64_t > uint64_vector; diff --git a/interop/model/metrics/tile_metric.h b/interop/model/metrics/tile_metric.h index 4cd486fd9..7408187a3 100644 --- a/interop/model/metrics/tile_metric.h +++ b/interop/model/metrics/tile_metric.h @@ -14,6 +14,7 @@ #include #include #include +#include "interop/util/math.h" #include "interop/io/format/generic_layout.h" #include "interop/io/layout/base_metric.h" #include "interop/model/metric_base/base_cycle_metric.h" @@ -34,16 +35,16 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param read read number * @param percent_aligned percent of PhiX aligned in read * @param percent_phasing percent phasing - * @param percent_prePhasing percent pre-phasing + * @param percent_prephasing percent pre-phasing */ read_metric(const uint_t read = 0, const float percent_aligned = std::numeric_limits::quiet_NaN(), const float percent_phasing = std::numeric_limits::quiet_NaN(), - const float percent_prePhasing = std::numeric_limits::quiet_NaN()) : + const float percent_prephasing = std::numeric_limits::quiet_NaN()) : m_read(read), m_percent_aligned(percent_aligned), m_percent_phasing(percent_phasing), - m_percent_prephasing(percent_prePhasing) + m_percent_prephasing(percent_prephasing) { } @@ -113,6 +114,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics float m_percent_aligned; float m_percent_phasing; float m_percent_prephasing; + template + friend struct io::generic_layout; }; /** Tile metrics @@ -134,25 +137,26 @@ namespace illumina { namespace interop { namespace model { namespace metrics typedef std::vector read_metric_vector; /** Define a const read iterator */ typedef read_metric_vector::const_iterator const_iterator; - /** Read metric type - */ + /** Read metric type */ typedef read_metric read_metric_type; + private: + typedef read_metric_vector::iterator read_iterator; public: /** Constructor */ tile_metric() : metric_base::base_metric(0, 0), - m_cluster_density(0), - m_cluster_density_pf(0), - m_cluster_count(0), - m_cluster_count_pf(0) + m_cluster_density(std::numeric_limits::quiet_NaN()), + m_cluster_density_pf(std::numeric_limits::quiet_NaN()), + m_cluster_count(std::numeric_limits::quiet_NaN()), + m_cluster_count_pf(std::numeric_limits::quiet_NaN()) { } /** Constructor */ tile_metric(const header_type&) : metric_base::base_metric(0, 0), - m_cluster_density(0), - m_cluster_density_pf(0), - m_cluster_count(0), - m_cluster_count_pf(0) + m_cluster_density(std::numeric_limits::quiet_NaN()), + m_cluster_density_pf(std::numeric_limits::quiet_NaN()), + m_cluster_count(std::numeric_limits::quiet_NaN()), + m_cluster_count_pf(std::numeric_limits::quiet_NaN()) { } /** Constructor @@ -389,6 +393,34 @@ namespace illumina { namespace interop { namespace model { namespace metrics return m_read_metrics.size(); } /* @} */ + /** Update the phasing/prephasing slope if they don't exist + * + * @param number read number + * @param phasing phasing slope + * @param prephasing prephasing slope + */ + void update_phasing_if_missing(const size_t number, const float phasing, const float prephasing) + { + for (read_iterator b = m_read_metrics.begin(); b != m_read_metrics.end(); ++b) + { + if (b->read() == static_cast(number)) + { + if(std::isnan(b->percent_phasing())) + { + b->percent_phasing(phasing); + } + if(std::isnan(b->percent_prephasing())) + { + b->percent_prephasing(prephasing); + } + return; + } + } + m_read_metrics.push_back(read_metric(static_cast(number), + std::numeric_limits::quiet_NaN(), + phasing, + prephasing)); + } /** Density of clusters for each tile (in clusters per mm2) * * @deprecated Will be removed in 1.1.x (use cluster_density instead) diff --git a/interop/model/plot/axes.h b/interop/model/plot/axes.h index 8f8d52201..d501532f6 100644 --- a/interop/model/plot/axes.h +++ b/interop/model/plot/axes.h @@ -7,6 +7,7 @@ */ #pragma once #include +#include "interop/util/lexical_cast.h" namespace illumina { namespace interop { namespace model { namespace plot { @@ -60,6 +61,33 @@ namespace illumina { namespace interop { namespace model { namespace plot { { return m_max; } + /** Clear the axis + */ + void clear() + { + m_min = 0.0f; + m_max = 0.0f; + m_label = ""; + } + + friend std::ostream& operator<<(std::ostream& out, const axis& data) + { + std::ios::fmtflags previous_state( out.flags() ); + out << std::setprecision(10) << data.m_min << "," << data.m_max << ","; + out << data.m_label << ","; + out.flags(previous_state); + return out; + } + friend std::istream& operator>>(std::istream& in, axis& data) + { + std::string tmp; + std::getline(in, tmp, ','); + data.m_min = util::lexical_cast(tmp); + std::getline(in, tmp, ','); + data.m_max = util::lexical_cast(tmp); + std::getline(in, data.m_label, ','); + return in; + } private: float m_min; @@ -167,6 +195,26 @@ namespace illumina { namespace interop { namespace model { namespace plot { { return m_y; } + /** Clear both axes + */ + void clear() + { + m_x.clear(); + m_y.clear(); + } + + friend std::ostream& operator<<(std::ostream& out, const axes& data) + { + out << data.m_x; + out << data.m_y; + return out; + } + friend std::istream& operator>>(std::istream& in, axes& data) + { + in >> data.m_x; + in >> data.m_y; + return in; + } private: axis m_x; diff --git a/interop/model/plot/bar_point.h b/interop/model/plot/bar_point.h index 2c0418cb0..95228960c 100644 --- a/interop/model/plot/bar_point.h +++ b/interop/model/plot/bar_point.h @@ -60,6 +60,23 @@ namespace illumina { namespace interop { namespace model { namespace plot { return 0; } + friend std::ostream& operator<<(std::ostream& out, const bar_point& data) + { + std::ios::fmtflags previous_state( out.flags() ); + const size_t precision = 10; + out << static_cast< const data_point& > (data); + out << std::setprecision(precision) << io::table::handle_nan(data.m_width) << ","; + out.flags(previous_state); + return out; + } + friend std::istream& operator>>(std::istream& in, bar_point& data) + { + in >> static_cast< data_point& > (data); + std::string tmp; + std::getline(in, tmp, ','); + data.m_width = util::lexical_cast(tmp); + return in; + } private: float m_width; diff --git a/interop/model/plot/candle_stick_point.h b/interop/model/plot/candle_stick_point.h index 7cca621f9..9e54a037f 100644 --- a/interop/model/plot/candle_stick_point.h +++ b/interop/model/plot/candle_stick_point.h @@ -130,6 +130,34 @@ namespace illumina { namespace interop { namespace model { namespace plot { return m_data_point_count; } + friend std::ostream& operator<<(std::ostream& out, const candle_stick_point& data) + { + const size_t precision = 10; + out << static_cast< const data_point& > (data); + out << std::setprecision(precision) << io::table::handle_nan(data.m_p25) << ","; + out << std::setprecision(precision) << io::table::handle_nan(data.m_p75) << ","; + out << std::setprecision(precision) << io::table::handle_nan(data.m_lower) << ","; + out << std::setprecision(precision) << io::table::handle_nan(data.m_upper) << ","; + out << data.m_data_point_count << ","; + out << data.m_outliers.size() << ","; + io::table::write_csv(out, data.m_outliers.begin(), data.m_outliers.end(), ','); + return out; + } + friend std::istream& operator>>(std::istream& in, candle_stick_point& data) + { + in >> static_cast< data_point& > (data); + std::string tmp; + + io::table::read_value(in, data.m_p25, tmp); + io::table::read_value(in, data.m_p75, tmp); + io::table::read_value(in, data.m_lower, tmp); + io::table::read_value(in, data.m_upper, tmp); + io::table::read_value(in, data.m_data_point_count, tmp); + const size_t num_outliers = io::table::read_value(in, tmp); + data.m_outliers.resize(num_outliers); + io::table::read_csv(in, data.m_outliers.begin(), data.m_outliers.end()); + return in; + } private: y_type m_p25; diff --git a/interop/model/plot/chart_data.h b/interop/model/plot/chart_data.h index 6dd8286d4..c2b0dda36 100644 --- a/interop/model/plot/chart_data.h +++ b/interop/model/plot/chart_data.h @@ -141,6 +141,26 @@ namespace illumina { namespace interop { namespace model { namespace plot { return m_axes; } + /** Clear the title and axis + */ + void clear() + { + m_axes.clear(); + m_title = ""; + } + + friend std::ostream& operator<<(std::ostream& out, const chart_data& data) + { + out << data.m_title << ","; + out << data.m_axes; + return out; + } + friend std::istream& operator>>(std::istream& in, chart_data& data) + { + std::getline(in, data.m_title, ','); + in >> data.m_axes; + return in; + } private: axes m_axes; diff --git a/interop/model/plot/data_point.h b/interop/model/plot/data_point.h index 96f38a57f..6aa331e6a 100644 --- a/interop/model/plot/data_point.h +++ b/interop/model/plot/data_point.h @@ -6,6 +6,9 @@ * @copyright GNU Public License. */ #pragma once +#include "interop/util/exception.h" +#include "interop/util/lexical_cast.h" +#include "interop/io/table/csv_format.h" namespace illumina { namespace interop { namespace model { namespace plot { @@ -86,6 +89,22 @@ namespace illumina { namespace interop { namespace model { namespace plot { m_y = y; m_x = x; } + friend std::ostream& operator<<(std::ostream& out, const data_point& data) + { + out << std::setprecision(10) << io::table::handle_nan(data.m_x) << ","; + out << std::setprecision(10) << io::table::handle_nan(data.m_y) << ","; + //out << "\n"; + return out; + } + friend std::istream& operator>>(std::istream& in, data_point& data) + { + std::string tmp; + std::getline(in, tmp, ','); + data.m_x = util::lexical_cast(tmp); + std::getline(in, tmp, ','); + data.m_y = util::lexical_cast(tmp); + return in; + } private: x_type m_x; diff --git a/interop/model/plot/data_point_collection.h b/interop/model/plot/data_point_collection.h index 4cb2ac3dd..0434a48a0 100644 --- a/interop/model/plot/data_point_collection.h +++ b/interop/model/plot/data_point_collection.h @@ -10,6 +10,7 @@ #include #include "interop/util/exception.h" #include "interop/util/assert.h" +#include "interop/model/model_exceptions.h" namespace illumina { namespace interop { namespace model { namespace plot { diff --git a/interop/model/plot/filter_options.h b/interop/model/plot/filter_options.h index c48c35941..5620874d6 100644 --- a/interop/model/plot/filter_options.h +++ b/interop/model/plot/filter_options.h @@ -8,17 +8,17 @@ #pragma once #ifdef _MSC_VER -#pragma warning(push) #pragma warning(disable:4290)// MSVC warns that it ignores the exception specification. #endif - #include "interop/model/metric_base/base_metric.h" +#include "interop/util/indirect_range_iterator.h" #include "interop/util/lexical_cast.h" #include "interop/constants/enums.h" #include "interop/logic/utils/metric_type_ext.h" #include "interop/logic/utils/channel.h" #include "interop/model/run/info.h" + namespace illumina { namespace interop { namespace model { namespace plot { @@ -39,11 +39,11 @@ namespace illumina { namespace interop { namespace model { namespace plot enum UseAll { /** All sentinel for id types */ - ALL_IDS = 0, + ALL_IDS = 0, /** All sentinel for channel types */ - ALL_CHANNELS = -1, + ALL_CHANNELS = -1, /** All sentinel for base types */ - ALL_BASES = -1 + ALL_BASES = -1 }; public: @@ -63,7 +63,7 @@ namespace illumina { namespace interop { namespace model { namespace plot filter_options(const constants::tile_naming_method naming_method, const id_t lane = ALL_IDS, const channel_t channel = ALL_CHANNELS,// Order based on RunInfo.xml - const dna_base_t base = (dna_base_t) ALL_BASES, + const dna_base_t base = static_cast(ALL_BASES), const id_t surface = ALL_IDS, const id_t read = ALL_IDS, const id_t cycle = ALL_IDS, @@ -82,6 +82,22 @@ namespace illumina { namespace interop { namespace model { namespace plot m_naming_method(naming_method) { } + public: + /** Reset all options to default values (except naming_method) + */ + void reset() + { + m_lane = ALL_IDS; + m_channel = ALL_CHANNELS; + m_base = static_cast(ALL_BASES); + m_surface = ALL_IDS; + m_read = ALL_IDS; + m_cycle = ALL_IDS; + m_tile_number = ALL_IDS; + m_swath = ALL_IDS; + m_section = ALL_IDS; + } + public: /** Test if the filter options are valid, if not throw an exception */ @@ -97,15 +113,25 @@ namespace illumina { namespace interop { namespace model { namespace plot INTEROP_THROW(model::invalid_filter_option, "Invalid tile naming method: does not match RunInfo.xml"); if(!all_lanes() && m_lane > run_info.flowcell().lane_count()) - INTEROP_THROW(model::invalid_filter_option, "Lane number exceeds total number of lanes" << m_lane << " > " << run_info.flowcell().lane_count()); + INTEROP_THROW(model::invalid_filter_option, + "Lane number exceeds total number of lanes: " + << m_lane << " > " << run_info.flowcell().lane_count()); if(is_specific_surface() && m_surface > run_info.flowcell().surface_count()) - INTEROP_THROW(model::invalid_filter_option, "Surface number exceeds total number of surfaces" << m_surface << " > " << run_info.flowcell().surface_count()); + INTEROP_THROW(model::invalid_filter_option, + "Surface number exceeds total number of surfaces: " + << m_surface << " > " << run_info.flowcell().surface_count()); if(!all_tile_numbers() && m_tile_number > run_info.flowcell().tile_count()) - INTEROP_THROW(model::invalid_filter_option, "Tile number exceeds total number of tile numbers" << m_tile_number << " > " << run_info.flowcell().tile_count()); + INTEROP_THROW(model::invalid_filter_option, + "Tile number exceeds total number of tile numbers: " + << m_tile_number << " > " << run_info.flowcell().tile_count()); if(!all_swaths() && m_swath > run_info.flowcell().swath_count()) - INTEROP_THROW(model::invalid_filter_option, "Swath number exceeds total number of swaths" << m_swath << " > " << run_info.flowcell().swath_count()); + INTEROP_THROW(model::invalid_filter_option, + "Swath number exceeds total number of swaths: " + << m_swath << " > " << run_info.flowcell().swath_count()); if(!all_sections() && m_section > run_info.flowcell().sections_per_lane()) - INTEROP_THROW(model::invalid_filter_option, "Section number exceeds total number of sections" << m_section << " > " << run_info.flowcell().sections_per_lane()); + INTEROP_THROW(model::invalid_filter_option, + "Section number exceeds total number of sections: " + << m_section << " > " << run_info.flowcell().sections_per_lane()); if(logic::utils::is_base_metric(type)) { if(!all_bases() && (m_base >= static_cast(constants::NUM_OF_BASES) || m_base < 0)) @@ -114,27 +140,38 @@ namespace illumina { namespace interop { namespace model { namespace plot if(logic::utils::is_cycle_metric(type)) { if(!all_cycles() && m_cycle > run_info.total_cycles()) - INTEROP_THROW(model::invalid_filter_option, "Cycle number exceeds total number of cycles" << m_cycle << " > " << run_info.total_cycles()); + INTEROP_THROW(model::invalid_filter_option, + "Cycle number exceeds total number of cycles: " + << m_cycle << " > " << run_info.total_cycles() + << " reads: " << run_info.reads().size()); } if(logic::utils::is_read_metric(type)) { if(!all_reads() && m_read > run_info.reads().size()) - INTEROP_THROW(model::invalid_filter_option, "Read number exceeds total number of reads" << m_read << " > " << run_info.reads().size()); + INTEROP_THROW(model::invalid_filter_option, + "Read number exceeds total number of reads: " + << m_read << " > " << run_info.reads().size()); } if(logic::utils::is_channel_metric(type)) { if(!all_channels() && static_cast(m_channel) >= run_info.channels().size()) - INTEROP_THROW(model::invalid_filter_option, "Channel number exceeds total number of channels" << m_channel << " > " << run_info.channels().size()); + INTEROP_THROW(model::invalid_filter_option, + "Channel number exceeds total number of channels: " + << m_channel << " > " << run_info.channels().size()); } if(!check_ignored) return; if(!logic::utils::is_base_metric(type) && !all_bases()) - INTEROP_THROW(model::invalid_filter_option, "Invalid filter option base for metric " << constants::to_string(type)); + INTEROP_THROW(model::invalid_filter_option, + "Invalid filter option base for metric " << constants::to_string(type)); if(!logic::utils::is_cycle_metric(type) && !all_cycles()) - INTEROP_THROW(model::invalid_filter_option, "Invalid filter option cycle for metric " << constants::to_string(type)); + INTEROP_THROW(model::invalid_filter_option, + "Invalid filter option cycle for metric " << constants::to_string(type)); if(!logic::utils::is_read_metric(type) && !all_reads()) - INTEROP_THROW(model::invalid_filter_option, "Invalid filter option read for metric " << constants::to_string(type)); + INTEROP_THROW(model::invalid_filter_option, + "Invalid filter option read for metric " << constants::to_string(type)); if(!logic::utils::is_channel_metric(type) && !all_channels()) - INTEROP_THROW(model::invalid_filter_option, "Invalid filter option channel for metric " << constants::to_string(type)); + INTEROP_THROW(model::invalid_filter_option, + "Invalid filter option channel for metric " << constants::to_string(type)); //all_cycles } /** Test if metric is a valid tile @@ -503,6 +540,15 @@ namespace illumina { namespace interop { namespace model { namespace plot return m_cycle; } + /** Get surface to display + * + * @return cycle + */ + id_t surface() const + { + return m_surface; + } + /** Get a description for the cycle filter option * * @return description for the cycle filter option @@ -559,6 +605,201 @@ namespace illumina { namespace interop { namespace model { namespace plot { return all_reads() ? "All Reads" : "Read " + util::lexical_cast(m_read); } + /** Get the tile naming convention + * + * @return tile naming convention + */ + constants::tile_naming_method naming_method()const + { + return m_naming_method; + } + + public: + /** Create an iterator that updates the current object + * + * flowcell - does not support lanes + * + * + * @param info run info + * @param metric_type metric type + * @param plot_type type of the plot + * @param keep_state keep the current state of the options + * @return iterator + */ + util::chain_range_iterator option_iterator(const model::run::info& info, + const constants::metric_type metric_type, + const constants::plot_types plot_type, + const bool keep_state=false) + { + const id_t lane_beg = !supports_lane(plot_type) || supports_all_lanes(plot_type) ? + static_cast(ALL_IDS) : 1; + const id_t lane_end = !supports_lane(plot_type) ? + static_cast(ALL_IDS) : static_cast(info.flowcell().lane_count()+1); + const channel_t channel_beg = !supports_channel(metric_type) || supports_all_channels(plot_type) ? + static_cast(ALL_CHANNELS) : 0; + const channel_t channel_end = !supports_channel(metric_type) ? + static_cast(ALL_CHANNELS) : + static_cast(info.channels().size()); + const dna_base_t base_beg = !supports_base(metric_type) || supports_all_bases(plot_type) ? + static_cast(ALL_BASES) : constants::A; + const dna_base_t base_end = !supports_base(metric_type) ? + static_cast(ALL_BASES) : constants::NUM_OF_BASES; + const id_t surface_beg = static_cast(ALL_IDS); // All surfaces is always supported + const id_t surface_end = !supports_surface(metric_type, info) ? + static_cast(ALL_IDS) : info.flowcell().surface_count()+1; + const id_t read_beg = !supports_read(metric_type, plot_type) || supports_all_reads(plot_type) ? + static_cast(ALL_IDS) : 1; + const id_t read_end = !supports_read(metric_type, plot_type) ? + static_cast(ALL_IDS) : static_cast(info.reads().size()+1); + const id_t cycle_beg = !supports_cycle(metric_type, plot_type) || supports_all_cycles(plot_type) ? + static_cast(ALL_IDS) : 1u; + const id_t cycle_end = !supports_cycle(metric_type, plot_type) ? + static_cast(ALL_IDS) : static_cast(info.total_cycles()+1); + const id_t swath_beg = static_cast(ALL_IDS); + const id_t swath_end = !supports_swath(plot_type) ? static_cast(ALL_IDS) : + static_cast(info.flowcell().swath_count()+1); + const id_t tile_beg = static_cast(ALL_IDS); + const id_t tile_end = !supports_tile(plot_type) ? static_cast(ALL_IDS) : + static_cast(info.flowcell().tile_count()+1); + util::abstract_range_iterator* iterators[] = + { + new util::indirect_range_iterator(m_lane, lane_beg, lane_end, !keep_state), + new util::indirect_range_iterator(m_channel, channel_beg, channel_end, !keep_state), + new util::indirect_range_iterator(m_base, base_beg, base_end, !keep_state), + new util::indirect_range_iterator(m_surface, surface_beg, surface_end, !keep_state), + new util::indirect_range_iterator(m_read, read_beg, read_end, !keep_state), + new util::indirect_range_iterator(m_cycle, cycle_beg, cycle_end, !keep_state), + new util::indirect_range_iterator(m_swath, swath_beg, swath_end, !keep_state), + new util::indirect_range_iterator(m_tile_number, tile_beg, tile_end, !keep_state) + }; + return util::chain_range_iterator(iterators); + } + + public: + /** Test if plot supports filtering by swath + * + * param plot_type type of plot + * @return true if metric supports filtering by swath + */ + bool supports_swath(const constants::plot_types /*plot_type*/)const + { + return false; + } + /** Test if plot supports filtering by tile + * + * param plot_type type of plot + * @return true if metric supports filtering by tile + */ + bool supports_tile(const constants::plot_types /*plot_type*/)const + { + return false; + } + /** Test if plot supports filtering by all lanes + * + * @param plot_type type of plot + * @return true if metric supports filtering by all lanes + */ + bool supports_all_lanes(const constants::plot_types plot_type)const + { + return plot_type != constants::SampleQCPlot; + } + /** Test if plot supports filtering by lane + * + * @param plot_type type of plot + * @return true if metric supports filtering by lane + */ + bool supports_lane(const constants::plot_types plot_type)const + { + return plot_type != constants::FlowcellPlot && + plot_type != constants::ByLanePlot; + } + /** Test if plot supports filtering by all bases + * + * @param plot_type plot type + * @return true if plot supports filtering by base + */ + bool supports_all_bases(const constants::plot_types plot_type)const + { + return plot_type == constants::ByCyclePlot; + } + /** Test if metric supports filtering by base + * + * @param metric_type metric type + * @return true if metric supports filtering by base + */ + bool supports_base(const constants::metric_type metric_type)const + { + return logic::utils::is_base_metric(metric_type); + } + /** Test if plot supports filtering by all channels + * + * @param plot_type plot type + * @return true if plot supports filtering all channels + */ + bool supports_all_channels(const constants::plot_types plot_type)const + { + return plot_type == constants::ByCyclePlot; + } + /** Test if metric supports filtering by channel + * + * @param metric_type metric type + * @return true if metric supports filtering by channel + */ + bool supports_channel(const constants::metric_type metric_type)const + { + return logic::utils::is_channel_metric(metric_type); + } + /** Test if plot supports filtering by all cycles + * + * @param plot_type plot type + * @return true if plot supports filtering by all cycles + */ + bool supports_all_cycles(const constants::plot_types plot_type)const + { + return plot_type == constants::QHistogramPlot; + } + /** Test if metric and plot combination supports filtering by cycle + * + * @param metric_type metric type + * @return true if metric supports filtering by cycle + */ + bool supports_cycle(const constants::metric_type metric_type, const constants::plot_types plot_type)const + { + return logic::utils::is_cycle_metric(metric_type) && + plot_type != constants::ByCyclePlot && + plot_type != constants::QHeatmapPlot; + } + /** Test if plot supports filtering by all reads + * + * @param plot_type plot type + * @return true if plot supports filtering by all reads + */ + bool supports_all_reads(const constants::plot_types plot_type)const + { + return plot_type == constants::QHistogramPlot; + } + /** Test if metric and plot combination supports filtering by read + * + * @param metric_type metric type + * @param plot_type type of plot + * @return true if metric supports filtering by read + */ + bool supports_read(const constants::metric_type metric_type, const constants::plot_types plot_type)const + { + return logic::utils::is_read_metric(metric_type) || plot_type == constants::QHistogramPlot; + } + /** Test if metric and run info combination supports filtering by surface + * + * @param metric_type metric type + * @param info run info + * @return true if metric supports filtering by surface + */ + bool supports_surface(const constants::metric_type metric_type, const model::run::info& info)const + { + if(metric_type == constants::UnknownMetricType) return false; + const size_t surface_count = info.flowcell().surface_count(); + return surface_count > 1; + } private: template @@ -579,6 +820,21 @@ namespace illumina { namespace interop { namespace model { namespace plot bool valid_read(const Metric &, const void*) const { return true; } + friend std::ostream& operator <<(std::ostream& out, const filter_options& options) + { + out << "Selected_filters_"; + if(!options.all_lanes()) out << "Lane_" << options.m_lane << "_"; + if(!options.all_channels()) out << "Channel_" << options.m_channel << "_"; + if(!options.all_bases()) out << "Base_" << options.m_base << "_"; + if(options.is_specific_surface()) out << "Surface_" << options.m_surface << "_"; + if(!options.all_reads()) out << "Read_" << options.m_read << "_"; + if(!options.all_cycles()) out << "Cycle_" << options.m_cycle << "_"; + if(!options.all_swaths()) out << "Swath_" << options.m_swath << "_"; + if(!options.all_tile_numbers()) out << "Tile_" << options.m_tile_number << "_"; + out << "_"; + return out; + } + private: id_t m_lane; @@ -596,6 +852,4 @@ namespace illumina { namespace interop { namespace model { namespace plot }}}} -#ifdef _MSC_VER -#pragma warning(pop) -#endif + diff --git a/interop/model/plot/flowcell_data.h b/interop/model/plot/flowcell_data.h index 197d5ccd6..b569946bf 100644 --- a/interop/model/plot/flowcell_data.h +++ b/interop/model/plot/flowcell_data.h @@ -7,8 +7,10 @@ */ #pragma once +#include #include "interop/util/exception.h" #include "interop/util/assert.h" +#include "interop/util/cstdint.h" #include "interop/model/plot/series.h" #include "interop/model/plot/axes.h" #include "interop/model/plot/heatmap_data.h" @@ -82,6 +84,19 @@ namespace illumina { namespace interop { namespace model { namespace plot m_tile_count = 0; } + /** Get tile id at index + * + * @param index index of id + * @return tile id + */ + ::uint32_t tile_at(const size_t index)const + { + INTEROP_ASSERTMSG(m_data != 0, "length: " << length()); + if (index >= length()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Tile index out of bounds"); + return m_data[index]; + } + /** Set data at given location in the flowcell * * @param lane_idx lane index @@ -118,6 +133,23 @@ namespace illumina { namespace interop { namespace model { namespace plot return m_data[index_of(lane_idx, loc)]; } + /** Get the tile id associated with the location + * + * @param lane_idx + * @param loc + * @return tile id + */ + ::uint32_t& tile_id(const size_t lane_idx, const size_t loc) throw(model::index_out_of_bounds_exception) + { + if (lane_idx >= lane_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Lane index out of bounds"); + if (loc >= column_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Location index out of bounds"); + INTEROP_ASSERT(index_of(lane_idx, loc) < length()); + INTEROP_ASSERT(m_data != 0); + return m_data[index_of(lane_idx, loc)]; + } + /** Set the axis * * @param plot_axis single axis @@ -254,6 +286,33 @@ namespace illumina { namespace interop { namespace model { namespace plot std::memset(reinterpret_cast(m_data), 0, sizeof(::uint32_t) * heatmap_data::length()); m_free = true; } + friend std::ostream& operator<<(std::ostream& out, const flowcell_data& data) + { + out << static_cast(data); + out << data.m_subtitle << ","; + out << data.m_swath_count << ","; + out << data.m_tile_count << ","; + for(size_t i=0, n=data.length();i>(std::istream& in, flowcell_data& data) + { + in >> static_cast(data); + std::string tmp; + std::getline(in, data.m_subtitle, ','); + std::getline(in, tmp, ','); + const size_t swath_count = util::lexical_cast(tmp); + std::getline(in, tmp, ','); + const size_t tile_count = util::lexical_cast(tmp); + data.resize(swath_count, tile_count); + for(size_t i=0, n=data.length();i(tmp); + } + return in; + } protected: /** Array of tile numbers for each tile */ diff --git a/interop/model/plot/heatmap_data.h b/interop/model/plot/heatmap_data.h index eb5904d10..461619e39 100644 --- a/interop/model/plot/heatmap_data.h +++ b/interop/model/plot/heatmap_data.h @@ -12,6 +12,7 @@ #include "interop/model/plot/series.h" #include "interop/model/plot/axes.h" #include "interop/model/plot/chart_data.h" +#include "interop/io/table/csv_format.h" namespace illumina { namespace interop { namespace model { namespace plot { @@ -101,6 +102,20 @@ namespace illumina { namespace interop { namespace model { namespace plot return m_data[idx]; } + /** Get value at given index + * + * @param idx index + * @return value + */ + float at(const size_t idx) const throw(model::index_out_of_bounds_exception) + { + if (idx >= length()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Index out of bounds"); + INTEROP_ASSERT(idx < m_num_rows*m_num_columns); + INTEROP_ASSERT(m_data != 0); + return m_data[idx]; + } + /** Get value at given row and column * * TODO: This should thrown an exception if wrong @@ -188,6 +203,7 @@ namespace illumina { namespace interop { namespace model { namespace plot } m_num_columns = 0; m_num_rows = 0; + chart_data::clear(); } /** Get the index of the row and column in the array * @@ -199,6 +215,24 @@ namespace illumina { namespace interop { namespace model { namespace plot { return row * m_num_columns + col; } + friend std::ostream& operator<<(std::ostream& out, const heatmap_data& data) + { + out << static_cast(data); + out << data.m_num_columns << ","; + out << data.m_num_rows << ","; + io::table::write_csv(out, data.m_data, data.m_data+data.length(), ','); + return out; + } + friend std::istream& operator>>(std::istream& in, heatmap_data& data) + { + in >> static_cast(data); + std::string tmp; + const size_t col_count = io::table::read_value(in, tmp); + const size_t row_count = io::table::read_value(in, tmp); + data.resize(row_count, col_count); + io::table::read_csv(in, data.m_data, data.m_data+data.length()); + return in; + } private: float* m_data; diff --git a/interop/model/plot/plot_data.h b/interop/model/plot/plot_data.h index cd4eaad66..449f85a8f 100644 --- a/interop/model/plot/plot_data.h +++ b/interop/model/plot/plot_data.h @@ -38,6 +38,7 @@ namespace illumina { namespace interop { namespace model { namespace plot { void clear() { m_series.clear(); + chart_data::clear(); } /** Resize collection to given size * @@ -105,6 +106,14 @@ namespace illumina { namespace interop { namespace model { namespace plot { { return m_series.size(); } + /** Check if object has points + * + * @return true if plot has points + */ + bool empty()const + { + return size()==0; + } public: /** Get iterator to start of collection @@ -123,6 +132,25 @@ namespace illumina { namespace interop { namespace model { namespace plot { { return m_series.end(); } + friend std::ostream& operator<<(std::ostream& out, const plot_data& data) + { + out << static_cast(data); + out << data.m_series.size() << ","; + for(size_t i=0;i>(std::istream& in, plot_data& data) + { + std::string tmp; + in >> static_cast(data); + std::getline(in, tmp, ','); + size_t n = util::lexical_cast(tmp); + data.m_series.resize(n); + for(size_t i=0;i> data.m_series[i]; + return in; + } private: series_collection_t m_series; diff --git a/interop/model/plot/series.h b/interop/model/plot/series.h index 4e58b5722..06d5ab5fe 100644 --- a/interop/model/plot/series.h +++ b/interop/model/plot/series.h @@ -50,7 +50,7 @@ namespace illumina { namespace interop { namespace model { namespace plot { * @param series_type type of the series */ series(const std::string& title="", - const std::string color="Black", + const std::string color="Blue", const series_types series_type=Candlestick) : m_title(title), m_color(color), @@ -98,6 +98,48 @@ namespace illumina { namespace interop { namespace model { namespace plot { m_options.push_back(option); } + friend std::ostream& operator<<(std::ostream& out, const series& data) + { + out << data.m_title << ","; + out << data.m_color << ","; + out << data.m_series_type << ","; + out << data.m_options.size() << ","; + for(size_t i=0;i>(std::istream& in, series& data) + { + std::string tmp; + std::getline(in, data.m_title, ','); + std::getline(in, data.m_color, ','); + std::getline(in, tmp, ','); + int series_type = util::lexical_cast(tmp); + data.m_series_type = static_cast(series_type); + + std::getline(in, tmp, ','); + size_t num_options = util::lexical_cast(tmp); + data.m_options.resize(num_options); + for(size_t i=0;i(tmp); + data.resize(num_points); + for(size_t i=0;i> data[i]; + } + return in; + } + private: std::string m_title; std::string m_color; diff --git a/interop/model/run/flowcell_layout.h b/interop/model/run/flowcell_layout.h index 22d14656f..e1f4e7ba3 100644 --- a/interop/model/run/flowcell_layout.h +++ b/interop/model/run/flowcell_layout.h @@ -155,6 +155,30 @@ namespace illumina { namespace interop { namespace model { namespace run { m_naming_method = naming_method; } + /** Set number of lanes + * + * @param lane_count number of lanes + */ + void lane_count(const uint_t lane_count) + { m_lane_count = lane_count; } + /** Set number of surfaces + * + * @param lane_count number of surfaces + */ + void surface_count(const uint_t surface_count) + { m_surface_count = surface_count; } + /** Set number of swathes + * + * @param swath_count number of swathes + */ + void swath_count(const uint_t swath_count) + { m_swath_count = swath_count; } + /** Set number of tiles + * + * @param tile_count number of tiles + */ + void tile_count(const uint_t tile_count) + { m_tile_count = tile_count; } private: tile_naming_method_t m_naming_method; diff --git a/interop/model/run/info.h b/interop/model/run/info.h index 71f12be48..4b9652ab2 100644 --- a/interop/model/run/info.h +++ b/interop/model/run/info.h @@ -80,8 +80,8 @@ namespace illumina { namespace interop { namespace model { namespace run * @param reads description of the reads */ info(const flowcell_layout &flowcell, - const str_vector_t &channels, - const read_vector_t &reads) : + const str_vector_t &channels=str_vector_t(), + const read_vector_t &reads=read_vector_t()) : m_version(0), m_flowcell(flowcell), m_channels(channels), @@ -222,6 +222,16 @@ namespace illumina { namespace interop { namespace model { namespace run void channels(const str_vector_t &channels) { m_channels = channels; } + /** Set the reads info + * + * @param read_vec reads info + */ + void reads(const read_vector_t& read_vec) + { + m_reads = read_vec; + m_total_cycle_count = total_cycles(); + } + /** Set the tile naming method * * @param naming_method tile naming method @@ -230,6 +240,12 @@ namespace illumina { namespace interop { namespace model { namespace run { m_flowcell.set_naming_method(naming_method); } + /** Set the layout of the flowcell + * + * @param flowcell layout of the flowcell + */ + void flowcell(const flowcell_layout & flowcell) + { m_flowcell = flowcell; } /** Get total number of cycles * diff --git a/interop/model/run/parameters.h b/interop/model/run/parameters.h index 5c7c3a8b5..5a5aaf2ee 100644 --- a/interop/model/run/parameters.h +++ b/interop/model/run/parameters.h @@ -1,6 +1,6 @@ /** Additional metadata describing a sequencing run * - * The files parsed by this class include: + * The InterOp files parsed by this class include: * - RunParameters.xml * * @file diff --git a/interop/model/run_metrics.h b/interop/model/run_metrics.h index 4887335bb..0402aa3ce 100644 --- a/interop/model/run_metrics.h +++ b/interop/model/run_metrics.h @@ -11,6 +11,7 @@ #include "interop/util/object_list.h" #include "interop/model/metric_base/metric_set.h" #include "interop/io/stream_exceptions.h" +#include "interop/io/metric_file_stream.h" #include "interop/model/run/info.h" #include "interop/model/run/parameters.h" @@ -65,24 +66,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics public: /** Define an id type */ typedef metric_base::base_metric::id_t id_t; - /** Define corrected intensity metric set */ - typedef metric_base::metric_set corrected_intensity_metric_set_t; - /** Define error metric set */ - typedef metric_base::metric_set error_metric_set_t; - /** Define extraction metric set */ - typedef metric_base::metric_set extraction_metric_set_t; - /** Define image metric set */ - typedef metric_base::metric_set image_metric_set_t; - /** Define index metric set */ - typedef metric_base::metric_set index_metric_set_t; - /** Define q-metric set */ - typedef metric_base::metric_set q_metric_set_t; - /** Define tile metric set */ - typedef metric_base::metric_set tile_metric_set_t; - /** Define collapsed q-metric set */ - typedef metric_base::metric_set q_collapsed_metric_set_t; - /** Define by lane q-metric set */ - typedef metric_base::metric_set q_by_lane_metric_set_t; /** Define a map of ids to a base metric */ typedef std::map tile_metric_map_t; /** Define a map of ids to a base cycle metric */ @@ -90,8 +73,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics public: /** Define set of ids */ - typedef error_metric_set_t::id_set_t id_set_t; - + typedef metric_base::metric_set::id_set_t id_set_t; public: /** Constructor @@ -214,169 +196,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ void set_naming_method(const constants::tile_naming_method naming_method); - public: - /** Get the set of corrected intensity metrics - * - * @return set of corrected intensity metrics - */ - corrected_intensity_metric_set_t &corrected_intensity_metric_set() - { - return get_set(); - } - - /** Get the set of error metrics - * - * @return set of error metrics - */ - error_metric_set_t &error_metric_set() - { - return get_set(); - } - - /** Get the set of extraction metrics - * - * @return set of extraction metrics - */ - extraction_metric_set_t &extraction_metric_set() - { - return get_set(); - } - - /** Get the set of image metrics - * - * @return set of image metrics - */ - image_metric_set_t &image_metric_set() - { - return get_set(); - } - - /** Get the set of index metrics - * - * @return set of index metrics - */ - index_metric_set_t &index_metric_set() - { - return get_set(); - } - - /** Get the set of quality metrics - * - * @return set of quality metrics - */ - q_metric_set_t &q_metric_set() - { - return get_set(); - } - - /** Get the set of collapsed quality metrics - * - * @return set of collapsed quality metrics - */ - q_collapsed_metric_set_t &q_collapsed_metric_set() - { - return get_set(); - } - - /** Get the set of by lane quality metrics - * - * @return set of by lane quality metrics - */ - q_by_lane_metric_set_t &q_by_lane_metric_set() - { - return get_set(); - } - - /** Get the set of tile metrics - * - * @return set of tile metrics - */ - tile_metric_set_t &tile_metric_set() - { - return get_set(); - } - - /** Set the set of tile metrics - * - * @param set set of tile metrics - */ - void tile_metric_set(const tile_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of collapsed q-metrics - * - * @param set set of collapsed q-metrics - */ - void q_collapsed_metric_set(const q_collapsed_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of by lane q-metrics - * - * @param set set of by lane q-metrics - */ - void q_by_lane_metric_set(const q_by_lane_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of q-metrics - * - * @param set set of q-metrics - */ - void q_metric_set(const q_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of index metrics - * - * @param set set of index metrics - */ - void index_metric_set(const index_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of image metrics - * - * @param set set of image metrics - */ - void image_metric_set(const image_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of extraction metrics - * - * @param set set of extraction metrics - */ - void extraction_metric_set(const extraction_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of error metrics - * - * @param set set of error metrics - */ - void error_metric_set(const error_metric_set_t &set) - { - get_set() = set; - } - - /** Set the set of corrected intensity metrics - * - * @param set set of corrected intensity metrics - */ - void corrected_intensity_metric_set(const corrected_intensity_metric_set_t &set) - { - get_set() = set; - } - public: /** Get information about the run * @@ -414,43 +233,61 @@ namespace illumina { namespace interop { namespace model { namespace metrics m_run_parameters = param; } - public: - /** Get a metric set + /** List all filenames for a specific metric * - * @return metric set + * @param files destination interop file names + * @param run_folder run folder location */ template - T &get() + void list_filenames(std::vector& files, const std::string& run_folder) + throw(invalid_run_info_exception) { - return m_metrics.get(); + typedef typename metric_base::metric_set_helper::metric_set_t metric_set_t; + const size_t last_cycle = run_info().total_cycles(); + if( last_cycle == 0 ) INTEROP_THROW(invalid_run_info_exception, "RunInfo is empty"); + io::list_interop_filenames< metric_set_t >(files, run_folder); } + public: + /** Set a given metric set + * + * @param metrics metric set + */ + template + void set(const T& metrics) + { + //static_assert( ) + m_metrics.get< T >() = metrics; + } /** Get a metric set * * @return metric set */ template - const T &get() const + typename metric_base::metric_set_helper::metric_set_t &get() { - return m_metrics.get(); + typedef typename metric_base::metric_set_helper::metric_set_t metric_set_t; + return m_metrics.get< metric_set_t >(); } - /** Get a metric set given a metric type + /** Get a metric set * * @return metric set */ template - metric_base::metric_set &get_set() + const typename metric_base::metric_set_helper::metric_set_t &get() const { - return m_metrics.get >(); + typedef typename metric_base::metric_set_helper::metric_set_t metric_set_t; + return m_metrics.get< metric_set_t >(); } /** Get a metric set given a metric type * + * @todo Remove. This is used by the SWIG binding because it is hard to wrap the return type of get<>() * @return metric set */ template - const metric_base::metric_set &get_set() const + metric_base::metric_set &get_metric_set() { return m_metrics.get >(); } @@ -484,7 +321,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * - Missing RunParameters.xml for non-legacy run folders * * @param run_folder run folder path - * @param valid_to_load list of metrics to load + * @param valid_to_load boolean vector indicating which files to load */ void read_metrics(const std::string &run_folder, const std::vector& valid_to_load) throw( io::file_not_found_exception, @@ -547,7 +384,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** Populate a map of valid tiles and cycles * - * @param map mapping between tile has and base_metric + * @param map mapping between tile has and base_cycle_metric */ void populate_id_map(cycle_metric_map_t &map) const; /** Sort the metrics by id diff --git a/interop/model/summary/index_count_summary.h b/interop/model/summary/index_count_summary.h index 8c7b04211..3497e587c 100644 --- a/interop/model/summary/index_count_summary.h +++ b/interop/model/summary/index_count_summary.h @@ -8,6 +8,7 @@ #pragma once #include #include "interop/util/math.h" +#include "interop/io/format/generic_layout.h" namespace illumina { namespace interop { namespace model { namespace summary { @@ -31,12 +32,12 @@ namespace illumina { namespace interop { namespace model { namespace summary { const std::string& index2="", const std::string& sample_id="", const std::string& project_name="", - const size_t count=0, + const ::uint64_t count=0, const float fraction_mapped=0.0f) : m_id(id), m_index1(index1), m_index2(index2), m_fraction_mapped(fraction_mapped), - m_count(count), + m_cluster_count(count), m_sample_id(sample_id), m_project_name(project_name) {} @@ -86,9 +87,9 @@ namespace illumina { namespace interop { namespace model { namespace summary { * * @return cluster count */ - size_t count()const + ::uint64_t cluster_count()const { - return m_count; + return m_cluster_count; } /** Sample id * @@ -110,12 +111,12 @@ namespace illumina { namespace interop { namespace model { namespace summary { public: /** Update the count * - * @param count count to add + * @param cluster_count count to add * @return reference to this object */ - index_count_summary& operator+=(const size_t count) + index_count_summary& operator+=(const ::uint64_t cluster_count) { - m_count += count; + m_cluster_count += cluster_count; return *this; } /** Update the fraction mapped from the total PF cluster count @@ -125,7 +126,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { void update_fraction_mapped(const double total_pf_cluster_count) { if(total_pf_cluster_count != 0.0f) - m_fraction_mapped = roundf(static_cast(m_count / total_pf_cluster_count * 100.0)*10000)/10000; + m_fraction_mapped = roundf(static_cast(m_cluster_count / total_pf_cluster_count * 100.0)*10000)/10000; } /** Compare two index count summaries by their ids * @@ -142,9 +143,13 @@ namespace illumina { namespace interop { namespace model { namespace summary { std::string m_index1; std::string m_index2; float m_fraction_mapped; - size_t m_count; + ::uint64_t m_cluster_count; std::string m_sample_id; std::string m_project_name; + + + template + friend struct io::generic_layout; }; }}}} diff --git a/interop/model/summary/index_flowcell_summary.h b/interop/model/summary/index_flowcell_summary.h index c131e69fe..58f47e2ce 100644 --- a/interop/model/summary/index_flowcell_summary.h +++ b/interop/model/summary/index_flowcell_summary.h @@ -10,6 +10,7 @@ #include "interop/util/exception.h" #include "interop/util/assert.h" #include "interop/model/model_exceptions.h" +#include "interop/io/format/generic_layout.h" #include "interop/model/summary/index_lane_summary.h" namespace illumina { namespace interop { namespace model { namespace summary @@ -142,9 +143,22 @@ namespace illumina { namespace interop { namespace model { namespace summary for(iterator b = begin(), e=end();b != e;++b) b->sort(); } + /** Clear the lane info + */ + void clear() + { + m_lane_summaries.clear(); + } + + friend std::ostream& operator<<(std::ostream& out, const index_flowcell_summary& summary); + friend std::istream& operator>>(std::istream& in, index_flowcell_summary& summary); private: lane_summary_vector_t m_lane_summaries; + + + template + friend struct io::generic_layout; }; }}}} diff --git a/interop/model/summary/index_lane_summary.h b/interop/model/summary/index_lane_summary.h index be2761e40..3da11ccb7 100644 --- a/interop/model/summary/index_lane_summary.h +++ b/interop/model/summary/index_lane_summary.h @@ -10,6 +10,7 @@ #include #include "interop/util/cstdint.h" #include "interop/util/exception.h" +#include "interop/io/format/generic_layout.h" #include "interop/model/summary/index_count_summary.h" namespace illumina { namespace interop { namespace model { namespace summary { @@ -60,6 +61,14 @@ namespace illumina { namespace interop { namespace model { namespace summary { } public: + /** Resize space for the number of indexes + * + * @param n number of indexes + */ + void resize(const size_type n) + { + m_count_summaries.resize(n); + } /** Reserve space for the number of indexes * * @param n number of indexes @@ -228,7 +237,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { * @param max_fraction_mapped maximum fraction of reads mapped * @param cv_fraction_mapped coefficient of variation of fraction of reads mapped */ - void set(const size_t total_mapped_reads, + void set(const ::uint64_t total_mapped_reads, const read_count_t pf_cluster_count_total, const read_count_t cluster_count_total, const float min_fraction_mapped, @@ -243,6 +252,18 @@ namespace illumina { namespace interop { namespace model { namespace summary { m_min_mapped_reads = (m_total_reads==0) ? 0 : roundf(min_fraction_mapped*10000)/10000; m_max_mapped_reads = (m_total_reads==0) ? 0 : roundf(max_fraction_mapped*10000)/10000; } + /** Clear the lane info + */ + void clear() + { + m_count_summaries.clear(); + m_total_reads = 0; + m_total_pf_reads = 0; + m_total_fraction_mapped_reads = 0; + m_mapped_reads_cv = 0; + m_min_mapped_reads = 0; + m_max_mapped_reads = 0; + } private: count_summary_vector_t m_count_summaries; @@ -254,6 +275,9 @@ namespace illumina { namespace interop { namespace model { namespace summary { float m_mapped_reads_cv; float m_min_mapped_reads; float m_max_mapped_reads; + + template + friend struct io::generic_layout; }; }}}} diff --git a/interop/model/summary/lane_summary.h b/interop/model/summary/lane_summary.h index e82eae10d..c3bba91a3 100644 --- a/interop/model/summary/lane_summary.h +++ b/interop/model/summary/lane_summary.h @@ -46,8 +46,10 @@ namespace illumina { namespace interop { namespace model { namespace summary /** Constructor * * @param lane lane number + * @param channel_count channel count */ - lane_summary(const size_t lane = 0) : + lane_summary(const size_t lane = 0, const size_t channel_count=0) : + stat_summary(channel_count), m_lane(lane), m_tile_count(0) { @@ -163,6 +165,24 @@ namespace illumina { namespace interop { namespace model { namespace summary return m_summary_by_surface.end(); } + /** Get random access iterator to start of summaries by lane + * + * @return random access iterator + */ + const_iterator begin()const + { + return m_summary_by_surface.begin(); + } + + /** Get random access iterator to end of summaries by lane + * + * @return random access iterator + */ + const_iterator end()const + { + return m_summary_by_surface.end(); + } + /** Resize the summary by lane vector * * @param n new size of summary by lane vector @@ -196,17 +216,6 @@ namespace illumina { namespace interop { namespace model { namespace summary return m_cycle_state; } - - /** Set statistics summarizing the cycle of each RTA state of tiles in the lane - * - * @param state statistics summarizing the cycle of each RTA state of tiles in the lane - */ - /*void cycle_state(cycle_state_summary& state) - { - m_cycle_state = state; - }*/ - - private: size_t m_lane; size_t m_tile_count; diff --git a/interop/model/summary/metric_summary.h b/interop/model/summary/metric_summary.h index a58ef0bb0..927295ed3 100644 --- a/interop/model/summary/metric_summary.h +++ b/interop/model/summary/metric_summary.h @@ -18,17 +18,16 @@ namespace illumina { namespace interop { namespace model { namespace summary { { public: /** Constructor - * */ - metric_summary() : - m_error_rate(0), - m_percent_aligned(0), - m_first_cycle_intensity(0), - m_percent_gt_q30(0), - m_yield_g(0), - m_projected_yield_g(0) - { - } + metric_summary(const size_t /*channel_count*/) : + m_error_rate(std::numeric_limits::quiet_NaN()), + m_percent_aligned(0), // TODO: Update logic to allow this to be NaN! + m_first_cycle_intensity(std::numeric_limits::quiet_NaN()), + m_percent_gt_q30(std::numeric_limits::quiet_NaN()), + m_yield_g(0), // TODO: Update logic to allow this to be NaN! + m_projected_yield_g(0) // TODO: Update logic to allow this to be NaN! + + {} public: /** @defgroup metric_summary Metric summary * @@ -78,6 +77,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { { return m_yield_g; } + /** Get the projected yield of teh run in gigabases * * @return projected yield of teh run in gigabases @@ -119,6 +119,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { { m_percent_gt_q30 = val; } + /** Set the yield of the run in gigabases * * @param val yield of the run in gigabases @@ -136,6 +137,12 @@ namespace illumina { namespace interop { namespace model { namespace summary { m_projected_yield_g = val; } + /** Resize the underlying data + */ + void resize(const size_t) + { + } + private: float m_error_rate; float m_percent_aligned; diff --git a/interop/model/summary/read_summary.h b/interop/model/summary/read_summary.h index 8ece68171..1310a72c2 100644 --- a/interop/model/summary/read_summary.h +++ b/interop/model/summary/read_summary.h @@ -41,9 +41,10 @@ namespace illumina { namespace interop { namespace model { namespace summary /** Constructor * * @param read read information + * @param channel_count number of channels */ - read_summary(const run::read_info &read = run::read_info()) : - m_read(read) + read_summary(const run::read_info &read = run::read_info(), const size_t channel_count=0) : + m_read(read), m_metric_summary(channel_count) { } diff --git a/interop/model/summary/run_summary.h b/interop/model/summary/run_summary.h index 0b5b67931..5dc75906f 100644 --- a/interop/model/summary/run_summary.h +++ b/interop/model/summary/run_summary.h @@ -39,7 +39,12 @@ namespace illumina { namespace interop { namespace model { namespace summary public: /** Constructor */ - run_summary() : m_surface_count(0), m_lane_count(0), m_read_count(0) + run_summary() : m_surface_count(0), + m_lane_count(0), + m_read_count(0), + m_channel_count(0), + m_total_summary(0), + m_nonindex_summary(0) { } @@ -49,12 +54,16 @@ namespace illumina { namespace interop { namespace model { namespace summary * @param end iterator to end of read collection * @param lane_count number of lanes on flowcell * @param surface_count number of surfaces on flowcell + * @param channel_count number of channels */ template - run_summary(I beg, I end, const size_t lane_count, const size_t surface_count) : + run_summary(I beg, I end, const size_t lane_count, const size_t surface_count, const size_t channel_count) : m_surface_count(surface_count), m_lane_count(lane_count), m_read_count(static_cast(std::distance(beg, end))), + m_channel_count(channel_count), + m_total_summary(channel_count), + m_nonindex_summary(channel_count), m_summary_by_read(beg, end) { preallocate_memory(); @@ -65,11 +74,18 @@ namespace illumina { namespace interop { namespace model { namespace summary * @param reads read info vector * @param lane_count number of lanes on flowcell * @param surface_count number of surfaces on flowcell + * @param channel_count number of channels */ - run_summary(const std::vector &reads, const size_t lane_count, const size_t surface_count) : + run_summary(const std::vector &reads, + const size_t lane_count, + const size_t surface_count, + const size_t channel_count) : m_surface_count(surface_count), m_lane_count(lane_count), m_read_count(reads.size()), + m_channel_count(channel_count), + m_total_summary(channel_count), + m_nonindex_summary(channel_count), m_summary_by_read(reads.begin(), reads.end()) { preallocate_memory(); @@ -82,23 +98,31 @@ namespace illumina { namespace interop { namespace model { namespace summary */ void initialize(const run::info& run_info) { - initialize(run_info.reads(), run_info.flowcell().lane_count(), run_info.flowcell().surface_count()); + initialize(run_info.reads(), run_info.flowcell().lane_count(), run_info.flowcell().surface_count(), run_info.channels().size()); } /** Initialize the run summary with the number of reads and lanes * * @param reads vector of reads * @param lane_count number of lanes * @param surface_count number of surfaces on flowcell + * @param channel_count number of channels */ - void initialize(const std::vector &reads, const size_t lane_count, const size_t surface_count) + void initialize(const std::vector &reads, + const size_t lane_count, + const size_t surface_count, + const size_t channel_count) { + m_total_summary = metric_summary(channel_count); + m_nonindex_summary = metric_summary(channel_count); + m_cycle_state = cycle_state_summary(); + m_channel_count = channel_count; m_read_count = reads.size(); m_summary_by_read.clear(); m_summary_by_read.reserve(reads.size()); for (size_t read = 0; read < reads.size(); ++read) - m_summary_by_read.push_back(read_summary(reads[read])); + m_summary_by_read.push_back(read_summary(reads[read], channel_count)); m_lane_count = lane_count; - m_surface_count=surface_count; + m_surface_count = surface_count; preallocate_memory(); } /** Copy reads to destination vector @@ -114,18 +138,24 @@ namespace illumina { namespace interop { namespace model { namespace summary private: void preallocate_memory() { + m_total_summary = metric_summary(m_channel_count); + m_nonindex_summary = metric_summary(m_channel_count); + m_cycle_state = cycle_state_summary(); for (iterator b = m_summary_by_read.begin(), e = m_summary_by_read.end(); b != e; ++b) { b->resize(m_lane_count); + b->summary().resize(m_channel_count); for (size_t lane = 0; lane < m_lane_count; ++lane) { b->at(lane).lane(lane + 1); + b->at(lane).resize_stat(m_channel_count); if(m_surface_count > 1) { b->at(lane).resize(m_surface_count); for (size_t surface = 0; surface < m_surface_count; ++surface) { b->at(lane).at(surface).surface(surface + 1); + b->at(lane).at(surface).resize_stat(m_channel_count); } } } @@ -233,14 +263,6 @@ namespace illumina { namespace interop { namespace model { namespace summary { return m_summary_by_read.end(); } - /** Clear the contents of the summary - */ - void clear() - { - m_summary_by_read.clear(); - m_lane_count = 0; - m_read_count = 0; - } public: /** Get number of lanes @@ -268,6 +290,14 @@ namespace illumina { namespace interop { namespace model { namespace summary { return m_surface_count; } + /** Get number of channels + * + * @return number of channels + */ + size_t channel_count() const + { + return m_channel_count; + } /** Set number of surfaces * @@ -353,6 +383,19 @@ namespace illumina { namespace interop { namespace model { namespace summary return m_cycle_state; } /** @} */ + /** Clear the contents of the summary + */ + void clear() + { + m_summary_by_read.clear(); + m_lane_count = 0; + m_read_count = 0; + m_surface_count = 0; + m_channel_count = 0; + m_total_summary = metric_summary(0); + m_nonindex_summary = metric_summary(0); + m_cycle_state = cycle_state_summary(); + } private: /** Resize the run summary with the number of reads and lanes @@ -364,10 +407,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { m_summary_by_read.clear(); m_summary_by_read.resize(m_read_count); - for (iterator b = m_summary_by_read.begin(), e = m_summary_by_read.end(); b != e; ++b) - { - b->resize(m_lane_count); - } + preallocate_memory(); } friend std::ostream& operator<<(std::ostream& out, const run_summary& summary); @@ -377,6 +417,8 @@ namespace illumina { namespace interop { namespace model { namespace summary size_t m_surface_count; size_t m_lane_count; size_t m_read_count; + size_t m_channel_count; + private: metric_summary m_total_summary; metric_summary m_nonindex_summary; diff --git a/interop/model/summary/stat_summary.h b/interop/model/summary/stat_summary.h index 6fd9eacd1..d2dda4210 100644 --- a/interop/model/summary/stat_summary.h +++ b/interop/model/summary/stat_summary.h @@ -28,13 +28,16 @@ namespace illumina { namespace interop { namespace model { namespace summary typedef metric_stat metric_stat_t; public: /** Constructor + * + * param channel_count number of channels */ - stat_summary() : + stat_summary(const size_t /*channel_count*/) : m_percent_gt_q30(std::numeric_limits::quiet_NaN()), - m_yield_g(0), - m_projected_yield_g(0), - m_reads(0), - m_reads_pf(0) + m_yield_g(0), // TODO: Update logic to allow this to be NaN! + m_projected_yield_g(0), // TODO: Update logic to allow this to be NaN! + m_reads(std::numeric_limits::quiet_NaN()), + m_reads_pf(std::numeric_limits::quiet_NaN()) + { } @@ -375,6 +378,13 @@ namespace illumina { namespace interop { namespace model { namespace summary m_first_cycle_intensity=stat; } + public: + /** Resize the underlying data + */ + void resize_stat(const size_t) + { + } + private: float m_percent_gt_q30; float m_yield_g; diff --git a/interop/model/summary/surface_summary.h b/interop/model/summary/surface_summary.h index d7b4080e8..3205b5ca0 100644 --- a/interop/model/summary/surface_summary.h +++ b/interop/model/summary/surface_summary.h @@ -28,8 +28,10 @@ namespace illumina { namespace interop { namespace model { namespace summary /** Constructor * * @param surface surface number + * @param channel_count number of channels */ - surface_summary(const size_t surface = 0) : + surface_summary(const size_t surface = 0, const size_t channel_count=0) : + stat_summary(channel_count), m_surface(surface), m_tile_count(0) { diff --git a/interop/util/constant_mapping.h b/interop/util/constant_mapping.h index 6125a82db..1cb9fbc2a 100644 --- a/interop/util/constant_mapping.h +++ b/interop/util/constant_mapping.h @@ -10,26 +10,64 @@ #pragma once -#include #include +#include +#include "interop/util/map.h" #include "interop/util/length_of.h" namespace illumina { namespace interop { namespace util { + namespace detail + { + /** Convert enum to proper hash + */ + struct enum_hash + { + /** Convert enum to hash + * + * @param val enum value + * @return hash value + */ + template + std::size_t operator()(const T val)const + { + return static_cast(val); + } +#if defined(__cplusplus) && __cplusplus >= 201103L + /** Convert enum to hash + * + * @param val enum value + * @return hash value + */ + std::size_t operator()(const std::string& val)const + { + return std::hash{}(val); + } +#endif + }; + } + /** A singleton that maps a key to a value */ template class constant_mapping { + typedef INTEROP_UNORDERED_HASHMAP(Key, Value, detail::enum_hash) constant_map_t; private: constant_mapping(const std::pair* pairs, size_t n) { - for (size_t i = 0; i < n; ++i) m_mapping[pairs[i].first] = pairs[i].second; + for (size_t i = 0; i < n; ++i) + { + m_mapping.insert(std::make_pair(pairs[i].first, pairs[i].second)); + } } constant_mapping(const std::pair* pairs, size_t n) { - for (size_t i = 0; i < n; ++i) m_mapping[pairs[i].second] = pairs[i].first; + for (size_t i = 0; i < n; ++i) + { + m_mapping.insert(std::make_pair(pairs[i].second, pairs[i].first)); + } } public: @@ -64,13 +102,13 @@ namespace illumina { namespace interop { namespace util */ const Value &get(const Key &key, const Value &default_value) const { - typename std::map::const_iterator it = m_mapping.find(key); + typename constant_map_t::const_iterator it = m_mapping.find(key); if (it == m_mapping.end()) return default_value; return it->second; } private: - std::map m_mapping; + constant_map_t m_mapping; }; /** Get the value corresponding to the key or default_value if none exists diff --git a/interop/util/cstdint.h b/interop/util/cstdint.h index 312eccc10..f8d0fe083 100644 --- a/interop/util/cstdint.h +++ b/interop/util/cstdint.h @@ -12,6 +12,8 @@ #pragma once #ifdef HAVE_STDINT_H #include +#elif HAVE_SYS_TYPES_H +#include #else #include "interop/util/pstdint.h" #endif diff --git a/interop/util/filesystem.h b/interop/util/filesystem.h index 2515286a2..a908934ce 100644 --- a/interop/util/filesystem.h +++ b/interop/util/filesystem.h @@ -24,6 +24,17 @@ namespace illumina { namespace interop { namespace io * @return proper os-dependent file path */ std::string combine(const std::string& path, const std::string& name); + /** Combine two directories or a directory and a filename into a file path + * + * This function provides a platform independent way to generate a file path. It currently supports most + * operating systems include Mac OSX, Windows and Linux/Unix. + * + * @param path1 string representing a file path 1 + * @param path2 string representing a file path 2 + * @param path3 string representing a file path 3 + * @return proper os-dependent file path + */ + std::string combine(const std::string& path1, const std::string& path2, const std::string& path3); /** Get the file name from a file path * * @param source full file path diff --git a/interop/util/indirect_range_iterator.h b/interop/util/indirect_range_iterator.h new file mode 100644 index 000000000..f108c8c48 --- /dev/null +++ b/interop/util/indirect_range_iterator.h @@ -0,0 +1,228 @@ +/** Iterator over a range of numbers + * + * This header provides facilities to manipulate files, directories and the paths that identify them. + * + * @file + * @date 10/27/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once + +namespace illumina { namespace interop { namespace util +{ + /** Abstract range iterator */ + class abstract_range_iterator + { + public: + /** Destructor */ + virtual ~abstract_range_iterator(){} + /** Increment the iterator to the next position in the range + * + * @return reference to iterator + */ + abstract_range_iterator& operator++() + { + advance(); + return *this; + } + /** Reset the iterator to the start of the range + */ + void operator()() + { + reset(); + } + /** Test if you can continue to iterate + * + * @return true if not done + */ + operator bool() const + { + return !is_done(); + } + + public: + /** Reset the iterator to the start of the range + */ + virtual void reset()=0; + /** Advance the iterator to the next element in the range + */ + virtual void advance()=0; + /** Test if you are at the end of the range + * + * @return true if you are done iterating, reached the end of the range + */ + virtual bool is_done()const=0; + }; + + /** Chain iterators into a recursive loop + * + * @note this class cannot be copied, data is moved! + */ + class chain_range_iterator : public abstract_range_iterator + { + public: + /** Constructor + * + * @param it1 iterator to first range + * @param it2 iterator to second range + */ + chain_range_iterator(abstract_range_iterator* it1=0, abstract_range_iterator* it2=0) : + m_iterator_1(it1), m_iterator_2(it2) + { + } + /** Constructor + * + * @param iterators array of iterators + * @param current current index to iterator + * @param dummy not used + * @return + */ + template + chain_range_iterator(abstract_range_iterator* (&iterators)[N], size_t current=0, size_t dummy=0) + { + (void)dummy; + m_iterator_1 = iterators[current]; + if( (current+2) < N ) + m_iterator_2 = new chain_range_iterator(iterators, current+1, 0); + else m_iterator_2 = iterators[current+1]; + } + /** Destructor */ + virtual ~chain_range_iterator() + { + delete m_iterator_1; + delete m_iterator_2; + } + + public: + /** Move constructor + * + * @note No copy is performed! Data is moved! + * @param other chain iterator to move data from + */ + chain_range_iterator(const chain_range_iterator& other) : m_iterator_1(other.m_iterator_1), m_iterator_2(other.m_iterator_2) + { + other.m_iterator_1 = 0; + other.m_iterator_2 = 0; + } + /** Move assignment operator + * + * @note No copy is performed! Data is moved! + * @param other chain iterator to move data from + * @return reference to this object + */ + chain_range_iterator& operator=(const chain_range_iterator& other) + { + delete m_iterator_1; + delete m_iterator_2; + m_iterator_1 = other.m_iterator_1; + m_iterator_2 = other.m_iterator_2; + other.m_iterator_1 = 0; + other.m_iterator_2 = 0; + return *this; + } + + public: + /** Reset the iterator to the start of the range + */ + void reset() + { + INTEROP_ASSERT(m_iterator_1 != 0); + INTEROP_ASSERT(m_iterator_2 != 0); + if(m_iterator_2 == 0 || m_iterator_1 == 0) return; + m_iterator_1->reset(); + m_iterator_2->reset(); + } + /** Advance the iterator to the next element in the range + */ + void advance() + { + INTEROP_ASSERT(m_iterator_1 != 0); + INTEROP_ASSERT(m_iterator_2 != 0); + if(m_iterator_2 == 0 || m_iterator_1 == 0) return; + m_iterator_1->advance(); + if(m_iterator_1->is_done()) + { + + m_iterator_2->advance(); + if(!m_iterator_2->is_done()) m_iterator_1->reset(); + } + } + /** Test if you are at the end of the range + * + * @return true if you are done iterating, reached the end of the range + */ + bool is_done()const + { + INTEROP_ASSERT(m_iterator_2 != 0); + if(m_iterator_2 == 0 || m_iterator_1 == 0) return true; + return m_iterator_1->is_done() && m_iterator_2->is_done(); + } + + private: + mutable abstract_range_iterator* m_iterator_1; + mutable abstract_range_iterator* m_iterator_2; + }; + /** Iterate over a range and update a reference to another value + */ + template + class indirect_range_iterator : public abstract_range_iterator + { + public: + /** Constructor + * + * @param rval reference to value to update + */ + indirect_range_iterator(T &rval) : m_value(&rval), + m_begin(static_cast(rval)), + m_end(static_cast(rval)) + { + reset(); + } + /** Constructor + * + * @param rval reference to value to update + * @param beg value at start of range + * @param end value at end of range + * @param reset_state reset current state + */ + indirect_range_iterator(T &rval, const T beg, const T end, const bool reset_state=true) : m_value(&rval), + m_begin(static_cast(beg)), + m_end(static_cast(end)) + { + if(reset_state) reset(); + } + /** Destructor */ + virtual ~indirect_range_iterator(){} + + public: + /** Reset the iterator to the start of the range + */ + void reset() + { + (*m_value) = static_cast(m_begin); + } + /** Advance the iterator to the next element in the range + */ + void advance() + { + const Storage sval = static_cast(*m_value)+1; + (*m_value) = static_cast(sval); + } + + /** Test if you are at the end of the range + * + * @return true if you are done iterating, reached the end of the range + */ + bool is_done() const + { + return (*m_value) >= static_cast(m_end); + } + + private: + T* m_value; + Storage m_begin; + Storage m_end; + }; + +}}} diff --git a/interop/util/length_of.h b/interop/util/length_of.h index f93188369..9b8cf985c 100644 --- a/interop/util/length_of.h +++ b/interop/util/length_of.h @@ -63,6 +63,18 @@ namespace illumina { namespace interop { namespace util size_t length_of(const T (&)[N]) { return N; } + /** Convert a stack array into a std::vector + * + * @tparam T type held by array + * @tparam N number of elements in array + * @param vals stack array + * @return std::vector + */ + template + static std::vector to_vector(const T (&vals)[N]) + { + return std::vector(vals, vals + N); + } }}} diff --git a/interop/util/lexical_cast.h b/interop/util/lexical_cast.h index 70c43eae9..035a65968 100644 --- a/interop/util/lexical_cast.h +++ b/interop/util/lexical_cast.h @@ -15,6 +15,7 @@ #include #include #include +#include "interop/util/math.h" #include "interop/util/type_traits.h" namespace illumina { namespace interop { namespace util @@ -40,51 +41,54 @@ namespace illumina { namespace interop { namespace util */ static Destination cast(const std::string &str) { - Destination val; - if (handle_special_float(str, val)) return val; + if (is_nan(str, static_cast(0))) + { + return nan_value(static_cast(0)); + } std::istringstream iss(str); + Destination val; iss >> val; return val; } private: - static bool handle_special_float(const std::string &str, float &val) + static double nan_value(double*) + { + return std::numeric_limits::quiet_NaN(); + } + static float nan_value(float*) + { + return std::numeric_limits::quiet_NaN(); + } + static Destination nan_value(void*) + { + return Destination(); + } + static bool is_nan(const std::string &str, double*) + { + return is_string_nan(str); + } + static bool is_nan(const std::string &str, float*) + { + return is_string_nan(str); + } + static bool is_nan(const std::string&, void*) { - if (str.length() == 3 && ::tolower(str[0]) =='n' && ::tolower(str[1]) == 'a' && ::tolower(str[2]) == 'n') - { - val = std::numeric_limits::quiet_NaN(); - return true; - } - if (str.length() == 4 && str[0] =='-' && ::tolower(str[1]) =='n' && ::tolower(str[2]) == 'a' && - ::tolower(str[3]) == 'n') - { - val = -std::numeric_limits::quiet_NaN(); - return true; - } return false; } - - static bool handle_special_float(const std::string &str, double &val) + static bool is_string_nan(const std::string &str) { if (str.length() == 3 && ::tolower(str[0]) =='n' && ::tolower(str[1]) == 'a' && ::tolower(str[2]) == 'n') { - val = std::numeric_limits::quiet_NaN(); return true; } if (str.length() == 4 && str[0] =='-' && ::tolower(str[1]) =='n' && ::tolower(str[2]) == 'a' && - ::tolower(str[3]) == 'n') + ::tolower(str[3]) == 'n') { - val = -std::numeric_limits::quiet_NaN(); return true; } return false; } - - template - static bool handle_special_float(const std::string &, T &) - { - return false; - } }; /** Specialization that casts a string to an arbitrary type @@ -246,7 +250,8 @@ namespace illumina { namespace interop { namespace util if (width > -1) oss << std::setw(width); if (precision > -1) oss << std::setprecision(precision); if (fill != 0) oss << std::setfill(fill); - oss << val; + if(std::isnan(val)) oss << std::numeric_limits::quiet_NaN(); + else oss << val; return oss.str(); } diff --git a/interop/util/map.h b/interop/util/map.h new file mode 100644 index 000000000..dff1d743e --- /dev/null +++ b/interop/util/map.h @@ -0,0 +1,23 @@ +/** Define generic ordered and unordered maps for both C++11 and C++98 + * + * @todo include this everywhere isnan is used + * + * @file + * @date 11/7/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once + +#if defined(__cplusplus) && __cplusplus < 201103L +# include +# define INTEROP_ORDERED_MAP(key_t, value_t) std::map +# define INTEROP_UNORDERED_MAP(key_t, value_t) std::map +# define INTEROP_UNORDERED_HASHMAP(key_t, value_t, hash_t) std::map +#else +# include +# include +# define INTEROP_ORDERED_MAP(key_t, value_t) std::map +# define INTEROP_UNORDERED_MAP(key_t, value_t) std::unordered_map +# define INTEROP_UNORDERED_HASHMAP(key_t, value_t, hash_t) std::unordered_map +#endif diff --git a/interop/util/math.h b/interop/util/math.h index 45c731023..dd6fc8560 100644 --- a/interop/util/math.h +++ b/interop/util/math.h @@ -11,7 +11,7 @@ #pragma once #include -#if !defined(HAVE_STD_ISNAN) +#if defined(HAVE_NO_STD_ISNAN) # if defined(HAVE_ISNAN) # include #elif defined(HAVE___ISNAN) diff --git a/interop/util/timer.h b/interop/util/timer.h new file mode 100644 index 000000000..91c221565 --- /dev/null +++ b/interop/util/timer.h @@ -0,0 +1,26 @@ +/** Scopped timer + * + * @file + * @date 11/7/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#pragma once +#include + +namespace illumina { namespace interop { namespace util +{ + + class scoped_timer + { + public: + scoped_timer(double& duration) : m_start(std::clock()), m_duration(duration){} + ~scoped_timer(){m_duration = (std::clock()-m_start) / static_cast(CLOCKS_PER_SEC);} + private: + std::clock_t m_start; + double& m_duration; + + }; + +}}} diff --git a/interop/util/type_traits.h b/interop/util/type_traits.h index 54e674091..002463e20 100644 --- a/interop/util/type_traits.h +++ b/interop/util/type_traits.h @@ -99,6 +99,10 @@ namespace illumina { namespace interop value = false }; }; + /** Test if type is a raw pointer */ + template struct is_pointer : false_type {}; + /** Test if type is a raw pointer */ + template struct is_pointer : true_type {}; /** Test if type has a const qualifier */ template struct is_const : false_type {}; /** Test if type has a const qualifier */ diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt index 43b1c2482..7e060e754 100644 --- a/src/apps/CMakeLists.txt +++ b/src/apps/CMakeLists.txt @@ -1,6 +1,13 @@ function(add_application _target _source_files) - add_executable(${_target} ${_source_files} inc/application.h inc/plot_options.h) + + if(MSVC) + set(SWIG_VERSION_INFO "${CMAKE_CURRENT_BINARY_DIR}/${_target}_version.rc") + set(LIB_NAME "${_target}") # Used to configure the version info file + configure_file(${CMAKE_SOURCE_DIR}/cmake/version.rc.in ${SWIG_VERSION_INFO} @ONLY) # Requires: LIB_NAME, VERSION_LIST and VERSION + endif() + + add_executable(${_target} ${_source_files} inc/application.h inc/plot_options.h ${SWIG_VERSION_INFO}) target_link_libraries(${_target} ${INTEROP_LIB}) if(COMPILER_IS_GNUCC_OR_CLANG) @@ -21,6 +28,8 @@ function(add_application _target _source_files) endif() endfunction() +set(SWIG_VERSION_INFO "") + add_application(interop2csv interop2csv.cpp) add_application(cyclesim cyclesim.cpp) add_application(summary summary.cpp) diff --git a/src/apps/cyclesim.cpp b/src/apps/cyclesim.cpp index 16d1a3500..1c3fd93df 100644 --- a/src/apps/cyclesim.cpp +++ b/src/apps/cyclesim.cpp @@ -43,14 +43,15 @@ #include "inc/application.h" using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::metric_base; using namespace illumina::interop; -typedef model::metrics::run_metrics::corrected_intensity_metric_set_t corrected_intensity_metric_set_t; -typedef model::metrics::run_metrics::tile_metric_set_t tile_metric_set_t; -typedef model::metrics::run_metrics::error_metric_set_t error_metric_set_t; -typedef model::metrics::run_metrics::extraction_metric_set_t extraction_metric_set_t; -typedef model::metrics::run_metrics::q_metric_set_t q_metric_set_t; -typedef model::metrics::run_metrics::image_metric_set_t image_metric_set_t; +typedef metric_set< corrected_intensity_metric > corrected_intensity_metric_set_t; +typedef metric_set< tile_metric > tile_metric_set_t; +typedef metric_set< error_metric > error_metric_set_t; +typedef metric_set< extraction_metric > extraction_metric_set_t; +typedef metric_set< q_metric > q_metric_set_t; +typedef metric_set< image_metric > image_metric_set_t; /** Write a help message to the output stream * @@ -205,7 +206,7 @@ int copy_tile_metrics(const std::string& input, const std::string& output, const metric_array_t subset; - for(const_iterator beg = metrics.metrics().begin(), end=metrics.metrics().end();beg != end;++beg) + for(const_iterator beg = metrics.begin(), end=metrics.end();beg != end;++beg) { read_metric_vector reads; for(const_read_iterator rbeg = beg->read_metrics().begin(), rend=beg->read_metrics().end();rbeg != rend;++rbeg) @@ -255,7 +256,7 @@ int copy_cycle_metrics(const std::string& input, const std::string& output, cons metric_array_t subset; - for(const_iterator beg = metrics.metrics().begin(), end=metrics.metrics().end();beg != end;++beg) + for(const_iterator beg = metrics.begin(), end=metrics.end();beg != end;++beg) { if(beg->tile()%2==0 && max_cycle > 1) { diff --git a/src/apps/dumpbin.cpp b/src/apps/dumpbin.cpp index 189450898..7aa2faf14 100644 --- a/src/apps/dumpbin.cpp +++ b/src/apps/dumpbin.cpp @@ -70,7 +70,8 @@ struct metric_writer out << ::int16_t(*sit); size_t char_count = length_of(::int16_t(*sit)); - for (++sit; sit != buffer.end(); ++sit, ++char_count) { + for (++sit; sit != buffer.end(); ++sit, ++char_count) + { const ::int16_t val = ::int16_t(*sit); char_count += 1 + length_of(val); if (char_count > k_max_line) { @@ -116,8 +117,9 @@ struct subset_copier { m_run.get() = MetricSet(metrics, metrics.version()); const size_t total = std::min(m_total, metrics.size()); + for(size_t i=0;i().insert(metrics.metrics()[i]); + m_run.get().insert(metrics.at(i)); } private: run_metrics& m_run; @@ -144,7 +146,15 @@ int main(int argc, char** argv) int ret = read_run_metrics(argv[i], run); if(ret != SUCCESS) return ret; subset_copier copy_subset(subset, subset_count); - run.metrics_callback(copy_subset); + try + { + run.metrics_callback(copy_subset); + } + catch(const std::exception& ex) + { + std::cerr << ex.what() << std::endl; + return UNEXPECTED_EXCEPTION; + } try { subset.metrics_callback(write_metrics); } diff --git a/src/apps/imaging_table.cpp b/src/apps/imaging_table.cpp index 6af20e9b0..299e024c4 100644 --- a/src/apps/imaging_table.cpp +++ b/src/apps/imaging_table.cpp @@ -42,6 +42,7 @@ int main(int argc, char** argv) std::cout << "# Version: " << INTEROP_VERSION << std::endl; std::vector valid_to_load; + logic::table::list_imaging_table_metrics_to_load(valid_to_load); for (int i = 1; i < argc; i++) { @@ -50,6 +51,7 @@ int main(int argc, char** argv) int ret = read_run_metrics(argv[i], run, valid_to_load); if (ret != SUCCESS) return ret; model::table::imaging_table table; + try { logic::table::create_imaging_table(run, table); diff --git a/src/apps/inc/application.h b/src/apps/inc/application.h index 20e4ff352..91e58fdd7 100644 --- a/src/apps/inc/application.h +++ b/src/apps/inc/application.h @@ -39,7 +39,9 @@ enum exit_codes * @param metrics run metrics * @return exit code */ -inline int read_run_metrics(const char* filename, illumina::interop::model::metrics::run_metrics& metrics) +inline int read_run_metrics(const char* filename, + illumina::interop::model::metrics::run_metrics& metrics, + const bool check_empty=true) { using namespace illumina::interop; using namespace illumina::interop::model; @@ -67,7 +69,7 @@ inline int read_run_metrics(const char* filename, illumina::interop::model::metr std::cerr << ex.what() << std::endl; return UNEXPECTED_EXCEPTION; } - if(metrics.empty()) + if(check_empty && metrics.empty()) { std::cerr << "No InterOp files found" << std::endl; return EMPTY_INTEROP; @@ -85,7 +87,8 @@ inline int read_run_metrics(const char* filename, illumina::interop::model::metr */ inline int read_run_metrics(const char* filename, illumina::interop::model::metrics::run_metrics& metrics, - const std::vector& valid_to_load) + const std::vector& valid_to_load, + const bool check_empty=true) { using namespace illumina::interop; using namespace illumina::interop::model; @@ -113,7 +116,7 @@ inline int read_run_metrics(const char* filename, std::cerr << ex.what() << std::endl; return UNEXPECTED_EXCEPTION; } - if(metrics.empty()) + if(check_empty && metrics.empty()) { std::cerr << "No InterOp files found" << std::endl; return EMPTY_INTEROP; diff --git a/src/apps/interop2csv.cpp b/src/apps/interop2csv.cpp index afad3b7ad..8b24e77de 100644 --- a/src/apps/interop2csv.cpp +++ b/src/apps/interop2csv.cpp @@ -115,15 +115,8 @@ #include #include #include "interop/io/metric_file_stream.h" -#include "interop/model/metrics/q_metric.h" -#include "interop/model/metrics/q_by_lane_metric.h" -#include "interop/model/metrics/q_collapsed_metric.h" -#include "interop/model/metrics/tile_metric.h" -#include "interop/model/metrics/error_metric.h" -#include "interop/model/metrics/corrected_intensity_metric.h" -#include "interop/model/metrics/extraction_metric.h" -#include "interop/model/metrics/image_metric.h" -#include "interop/model/metrics/index_metric.h" +#include "interop/model/run_metrics.h" +#include "interop/logic/utils/enums.h" #include "interop/logic/metric/q_metric.h" #include "interop/version.h" #include "inc/application.h" @@ -169,6 +162,9 @@ int main(int argc, char** argv) return 1; } + // TODO: Fix up this application! + std::cerr << "WARNING: This program is a debug utility and not currently supported. See the docs." << std::endl; + for(int i=1;iindex_seq() << "," << index_beg->sample_id() << "," << index_beg->sample_proj() << "," << index_beg->count() << "\n"; + out << index_beg->index_seq() << "," << index_beg->sample_id() << "," << index_beg->sample_proj() << "," << index_beg->cluster_count() << "\n"; } } diff --git a/src/apps/summary.cpp b/src/apps/summary.cpp index aeae5e19f..d6071786a 100644 --- a/src/apps/summary.cpp +++ b/src/apps/summary.cpp @@ -36,6 +36,7 @@ #include #include #include "interop/util/math.h" +#include "interop/util/length_of.h" #include "interop/io/metric_file_stream.h" #include "interop/logic/summary/run_summary.h" #include "interop/version.h" @@ -127,15 +128,6 @@ void print_array(std::ostream& out, I beg, I end, const size_t width, const char out.flags(f); out << std::endl; } -/** Get number of elements in stack array - * - * @return number of elements - */ -template -size_t size_of(const char*(&)[N]) -{ - return N; -} /** Take a array of strings and print them using a fixed width * * @param out output stream @@ -210,22 +202,23 @@ std::string format(const model::run::cycle_range& rng) } void summarize(const metric_summary& summary, std::vector& values) { - INTEROP_ASSERT(values.size() >= 7); // format(value, width in spaces, number of values after decimal, multiplier) - values[1] = util::format(summary.yield_g(), 3, 2); - values[2] = util::format(summary.projected_yield_g(), 3, 2); - values[3] = util::format(summary.percent_aligned(), 3, 2); - values[4] = util::format(summary.error_rate(), 3, 2); - values[5] = util::lexical_cast(long(summary.first_cycle_intensity()+0.5)); - values[6] = util::format(summary.percent_gt_q30(), 3, 2); + size_t i=1; + values[i++] = util::format(summary.yield_g(), 3, 2); + values[i++] = util::format(summary.projected_yield_g(), 3, 2); + values[i++] = util::format(summary.percent_aligned(), 3, 2); + values[i++] = util::format(summary.error_rate(), 3, 2); + values[i++] = util::lexical_cast(long(summary.first_cycle_intensity()+0.5)); + values[i++] = util::format(summary.percent_gt_q30(), 3, 2); + if(i != values.size()) INTEROP_THROW(std::runtime_error, "There is a bug in the program, columns do not match header"); } -void summarize(const surface_summary& summary, std::vector& values) +void summarize(const surface_summary& summary, std::vector& values, const size_t lane) { - INTEROP_ASSERT(values.size() >= 16); size_t i=0; - values[i++] = "-"; + values[i++] = util::lexical_cast(lane); values[i++] = util::lexical_cast(summary.surface()); values[i++] = util::lexical_cast(summary.tile_count()); + // format(value, width in spaces, number of values after decimal, multiplier) values[i++] = format(summary.density(), 0, 0, 1e3); values[i++] = format(summary.percent_pf(), 0, 2); @@ -241,14 +234,16 @@ void summarize(const surface_summary& summary, std::vector& values) values[i++] = format(summary.error_rate_75(), 0, 2); values[i++] = format(summary.error_rate_100(), 0, 2); values[i++] = format(summary.first_cycle_intensity(), 0, 0); + INTEROP_ASSERT(i==values.size()); + if(i != values.size()) INTEROP_THROW(std::runtime_error, "There is a bug in the program, columns do not match header"); } void summarize(const lane_summary& summary, std::vector& values) { - INTEROP_ASSERT(values.size() >= 16); size_t i=0; values[i++] = util::lexical_cast(summary.lane()); values[i++] = "-"; values[i++] = util::lexical_cast(summary.tile_count()); + // format(value, width in spaces, number of values after decimal, multiplier) values[i++] = format(summary.density(), 0, 0, 1e3); values[i++] = format(summary.percent_pf(), 0, 2); @@ -264,6 +259,8 @@ void summarize(const lane_summary& summary, std::vector& values) values[i++] = format(summary.error_rate_75(), 0, 2); values[i++] = format(summary.error_rate_100(), 0, 2); values[i++] = format(summary.first_cycle_intensity(), 0, 0); + INTEROP_ASSERT(i==values.size()); + if(i != values.size()) INTEROP_THROW(std::runtime_error, "There is a bug in the program, columns do not match header"); } std::string format_read(const run::read_info& read) { @@ -275,7 +272,7 @@ void print_summary(std::ostream& out, const run_summary& summary) const size_t width=15; const char* read_header[] = {"Level", "Yield", "Projected Yield", "Aligned", "Error Rate", "Intensity C1", "%>=Q30"}; print_array(out, read_header, width); - std::vector values(size_of(read_header)); + std::vector values(util::length_of(read_header)); INTEROP_ASSERT(values.size()>=1); for(size_t read=0;read=Q30", "Yield", "Cycles Error", "Aligned", "Error", "Error (35)", "Error (75)", "Error (100)", "Intensity C1" }; - values.resize(size_of(lane_header)); + values.resize(util::length_of(lane_header)); for(size_t read=0;read::metric_array_t::const_iterator beg = tile_metric_set.metrics().begin(), end = tile_metric_set.metrics().end();beg != end;++beg) + for(metric_set::metric_array_t::const_iterator beg = tile_metric_set.begin(), end = tile_metric_set.end();beg != end;++beg) { lane_summary_map[beg->lane()].cluster_count += beg->cluster_count(); lane_summary_map[beg->lane()].cluster_count_pf += beg->cluster_count_pf(); diff --git a/src/examples/example3.cpp b/src/examples/example3.cpp index 672f3a03d..8931117d2 100644 --- a/src/examples/example3.cpp +++ b/src/examples/example3.cpp @@ -32,15 +32,22 @@ int main(int argc, char** argv) if((ret = read_interop_file(argv[1], extraction_metric_set)) != 0) return ret; - std::time_t t = static_cast(extraction_metric_set.metrics()[0].date_time()); - std::tm* tm = std::gmtime(&t); - if(tm != 0) + try { - char buffer[80]; - std::strftime(buffer, 80, "%m/%d/%Y %r", tm); - std::cout << "Time: " << buffer << std::endl; + std::time_t t = static_cast(extraction_metric_set.at(0).date_time()); + std::tm *tm = std::gmtime(&t); + if (tm != 0) + { + char buffer[80]; + std::strftime(buffer, 80, "%m/%d/%Y %r", tm); + std::cout << "Time: " << buffer << std::endl; + } else std::cout << "Invalid time" << std::endl; + } + catch(const std::exception& ex) + { + std::cerr << ex.what() << std::endl; + return 1; } - else std::cout << "Invalid time" << std::endl; return 0; } diff --git a/src/examples/example4.cpp b/src/examples/example4.cpp index 1981af067..c4ee66384 100644 --- a/src/examples/example4.cpp +++ b/src/examples/example4.cpp @@ -45,16 +45,16 @@ int main(int argc, char** argv) } // Member function that takes a single argument - float avg = mean(corrected_intensity_metric_set.metrics().begin(), - corrected_intensity_metric_set.metrics().end(), + float avg = mean(corrected_intensity_metric_set.begin(), + corrected_intensity_metric_set.end(), op::const_member_function(constants::A, &corrected_intensity_metric::percentBase)); - float standard_dev = std::sqrt(variance(corrected_intensity_metric_set.metrics().begin(), - corrected_intensity_metric_set.metrics().end(), + float standard_dev = std::sqrt(variance(corrected_intensity_metric_set.begin(), + corrected_intensity_metric_set.end(), op::const_member_function(constants::A, &corrected_intensity_metric::percentBase))); // Member function that takes no arguments - float avg_no_call = mean(corrected_intensity_metric_set.metrics().begin(), - corrected_intensity_metric_set.metrics().end(), + float avg_no_call = mean(corrected_intensity_metric_set.begin(), + corrected_intensity_metric_set.end(), op::const_member_function(&corrected_intensity_metric::no_calls)); std::cout << "Mean of percent base for A " << avg << std::endl; std::cout << "Standard Deviation of percent base for A " << standard_dev << std::endl; diff --git a/src/ext/CMakeLists.txt b/src/ext/CMakeLists.txt index 30e847add..6fbb0150b 100644 --- a/src/ext/CMakeLists.txt +++ b/src/ext/CMakeLists.txt @@ -16,10 +16,10 @@ endif() # Need to enable SWIGWORDSIZE64 in SWIG when long is 64-bit # This is so stdint.i works properly ############################################################ -if(IS_INT64_LONG) +if(FORCE_X86) + set(SWIG_WORDSIZE_FLAG ";-DSWIGWORDSIZE32") +elseif(IS_INT64_LONG) set(SWIG_WORDSIZE_FLAG ";-DSWIGWORDSIZE64") -else() - set(SWIG_WORDSIZE_FLAG "") endif() if("${SIZE_T}" EQUAL 8) set(SWIG_WORDSIZE_FLAG "${SWIG_WORDSIZE_FLAG};-DHAVE_UINT64_SIZE_T") @@ -27,16 +27,14 @@ endif() include(${SWIG_USE_FILE}) set(CMAKE_SWIG_FLAGS "") -set(SWIG_SRCS_TEMP swig/run.i swig/metrics.i swig/comm.i swig/imaging.i swig/plot.i swig/summary.i) +set(SWIG_SRCS_TEMP swig/run.i swig/metrics.i swig/comm.i swig/table.i swig/plot.i swig/summary.i) set(SWIG_DEPS_TEMP swig/util/operator_overload.i swig/exceptions/exceptions_impl.i swig/exceptions/exceptions_csharp.i swig/arrays/arrays_csharp_impl.i swig/arrays/arrays_impl.i - swig/arrays/arrays_numpy_impl.i - swig/extends/extends_csharp.i - swig/extends/extends_impl.i) + swig/arrays/arrays_numpy_impl.i) foreach(SRC ${SWIG_SRCS_TEMP}) get_filename_component(SRC ${SRC} ABSOLUTE) @@ -59,6 +57,7 @@ endforeach() # Language options # https://github.com/SimpleITK/SimpleITK/blob/master/CMake/sitkLanguageOptions.cmake + if(ENABLE_CSHARP) add_subdirectory("csharp") endif() diff --git a/src/ext/csharp/CMakeLists.txt b/src/ext/csharp/CMakeLists.txt index dd4e145e3..9bf69f183 100644 --- a/src/ext/csharp/CMakeLists.txt +++ b/src/ext/csharp/CMakeLists.txt @@ -5,6 +5,7 @@ set(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_BINARY_DIR}) add_custom_target(csharp_lib) set_target_properties(csharp_lib PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) +set(SWIG_VERSION_INFO "") foreach(SRC ${SWIG_SRCS}) get_filename_component(MODULE ${SRC} NAME_WE) @@ -13,8 +14,13 @@ foreach(SRC ${SWIG_SRCS}) string(REGEX REPLACE "^.(.*)" "${FIRST_LETTER}\\1" NAMESPACE "${MODULE}") set_source_files_properties(${SRC} PROPERTIES SWIG_FLAGS "-namespace;Illumina.InterOp.${NAMESPACE};-module;c_csharp_${MODULE}${SWIG_WORDSIZE_FLAG}") set_source_files_properties(${SRC} PROPERTIES CPLUSPLUS ON) - set(SWIG_MODULE_c_csharp_${MODULE}_EXTRA_DEPS ${SWIG_DEPS} ) - swig_add_module(c_csharp_${MODULE} csharp ${SRC}) + if(MSVC) + set(SWIG_VERSION_INFO ${CMAKE_CURRENT_BINARY_DIR}/${MODULE}_version.rc) + set(LIB_NAME "${MODULE}") # Used to configure the version info file + configure_file(${CMAKE_SOURCE_DIR}/cmake/version.rc.in ${SWIG_VERSION_INFO} @ONLY) # Requires: LIB_NAME, VERSION_LIST and VERSION + endif() + set(SWIG_MODULE_c_csharp_${MODULE}_EXTRA_DEPS ${SWIG_DEPS} ${SWIG_VERSION_INFO}) + swig_add_module(c_csharp_${MODULE} csharp ${SRC} ${SWIG_VERSION_INFO}) set(SWIG_MODULE ${SWIG_MODULE_c_csharp_${MODULE}_REAL_NAME}) set(EXTRA_LINKER_FLAGS) if(MINGW) @@ -62,7 +68,9 @@ include(${CSHARP_USE_FILE}) set(SWIG_GEN_CSHARP_SOURCE_FILES ${CMAKE_SWIG_OUTDIR}/*.cs CACHE INTERNAL "C# source files generated by SWIG" FORCE) -csharp_add_library(csharp_interop ${CMAKE_SWIG_OUTDIR}/*.cs) +set(ASSEMBLY_INFO ${CMAKE_CURRENT_BINARY_DIR}/AssemblyInfo.cs) +configure_file(${CMAKE_SOURCE_DIR}/cmake/AssemblyInfo.cs.in ${ASSEMBLY_INFO} @ONLY) +csharp_add_library(csharp_interop ${CMAKE_SWIG_OUTDIR}/*.cs ${ASSEMBLY_INFO}) add_dependencies(csharp_interop ${SWIG_MODULE_LIST}) foreach(SRC ${SWIG_SRCS}) get_filename_component(MODULE ${SRC} NAME_WE) @@ -118,31 +126,12 @@ else() endif() endif() -function(create_nuspec_files _output _config _framework _source_files) - # C# Dlls should be packaged in a directory lib\\ - # C++ Shared Libraries should be packaged in a directory called build\ - # The following code generates this file list from the SWIG generated C++ shared libraries and the C# Dll - get_filename_component(DLL_NAME ${CSHARP_csharp_interop_BINARY} NAME) - set(_files ${${_output}}) - if(NOT "${_files}" STREQUAL "") - set(_files "${_files};") - endif() - set(_files "${_files}") - list(APPEND _files "") - foreach(SRC ${_source_files}) - get_filename_component(MODULE ${SRC} NAME_WE) - set(NUGET_FILE "\\\";target=\\\"build\\\\${_config}\\\";/>") - list(APPEND _files ${NUGET_FILE}) - endforeach() - set(${_output} ${_files} PARENT_SCOPE) -endfunction() - -set(NUGET_SYS_ID "${CMAKE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}${PACKAGE_SUFFIX}") +set(NUGET_SYS_ID "${CMAKE_SYSTEM_NAME}${PACKAGE_SUFFIX}") set(DOT_NET_FRAMEWORK "net40") -create_nuspec_files(NUGET_FILE_LIST "Debug" ${DOT_NET_FRAMEWORK} "${SWIG_SRCS}") -create_nuspec_files(NUGET_FILE_LIST "Release" ${DOT_NET_FRAMEWORK} "${SWIG_SRCS}") +get_filename_component(DLL_NAME ${CSHARP_csharp_interop_BINARY} NAME) if(BUILD_NUMBER) - set(INTEROP_VERSION_STR ${VERSION_SHORT}-alpha-${BUILD_NUMBER}) + MATH(EXPR VERSION_PATCHN "${VERSION_PATCH}+1") + set(INTEROP_VERSION_STR ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCHN}-alpha${BUILD_NUMBER}) else() set(INTEROP_VERSION_STR ${VERSION_SHORT}) endif() @@ -158,7 +147,8 @@ add_custom_target(nuspec -DCSHARP_VERSION=${CSHARP_VERSION} -DINTEROP_VERSION=${INTEROP_VERSION_STR} -DPLATFORM=${NUGET_SYS_ID} - -DNUGET_FILE_LIST="${NUGET_FILE_LIST}" + -DCONFIG="${CONFIG_DIR}" + -DSHARED_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}" -DCONFIG_INPUT_FILE=${CMAKE_SOURCE_DIR}/cmake/package.nuspec.in -DCONFIG_OUTPUT_FILE=$/../package.nuspec -P ${CMAKE_SOURCE_DIR}/cmake/ConfigureFile.cmake diff --git a/src/ext/swig/arrays/arrays_impl.i b/src/ext/swig/arrays/arrays_impl.i index 2c96fe899..a499a2616 100644 --- a/src/ext/swig/arrays/arrays_impl.i +++ b/src/ext/swig/arrays/arrays_impl.i @@ -25,7 +25,6 @@ %ignore vector_t::end; %ignore vector_t::operator[]; %apply size_t { vector_t::size_type }; -%ignore vector_t::at(const size_type) const; %enddef diff --git a/src/ext/swig/arrays/arrays_numpy_impl.i b/src/ext/swig/arrays/arrays_numpy_impl.i index 6dfdc6657..894917795 100644 --- a/src/ext/swig/arrays/arrays_numpy_impl.i +++ b/src/ext/swig/arrays/arrays_numpy_impl.i @@ -7,4 +7,9 @@ import_array(); %} -%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(unsigned char* buffer, size_t buffer_size)} \ No newline at end of file +%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(::uint8_t* buffer, size_t buffer_size)} +%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(unsigned char* buffer, size_t buffer_size)} +%apply (float* INPLACE_ARRAY1, int DIM1) {(float* buffer, size_t buffer_size)} +%apply (unsigned int* INPLACE_ARRAY1, int DIM1) {(::uint32_t* id_buffer, size_t id_buffer_size)} +%apply (float* INPLACE_ARRAY1, int DIM1) {(float* data_beg, const size_t n)} + diff --git a/src/ext/swig/arrays/numpy.i b/src/ext/swig/arrays/numpy.i index 58ad9e39b..2ddc11de7 100644 --- a/src/ext/swig/arrays/numpy.i +++ b/src/ext/swig/arrays/numpy.i @@ -249,9 +249,9 @@ } /* Given a PyArrayObject, check to see if it is contiguous. If so, - * return the input pointer and test_modifier it as not a new object. If it is + * return the input pointer and flag it as not a new object. If it is * not contiguous, create a new PyArrayObject using the original data, - * test_modifier it as a new object and return the pointer. + * flag it as a new object and return the pointer. */ PyArrayObject* make_contiguous(PyArrayObject* ary, int* is_new_object, @@ -276,9 +276,9 @@ } /* Given a PyArrayObject, check to see if it is Fortran-contiguous. - * If so, return the input pointer, but do not test_modifier it as not a new + * If so, return the input pointer, but do not flag it as not a new * object. If it is not Fortran-contiguous, create a new - * PyArrayObject using the original data, test_modifier it as a new object + * PyArrayObject using the original data, flag it as a new object * and return the pointer. */ PyArrayObject* make_fortran(PyArrayObject* ary, @@ -303,7 +303,7 @@ /* Convert a given PyObject to a contiguous PyArrayObject of the * specified type. If the input object is not a contiguous - * PyArrayObject, a new one will be created and the new object test_modifier + * PyArrayObject, a new one will be created and the new object flag * will be set. */ PyArrayObject* obj_to_array_contiguous_allow_conversion(PyObject* input, @@ -331,7 +331,7 @@ /* Convert a given PyObject to a Fortran-ordered PyArrayObject of the * specified type. If the input object is not a Fortran-ordered - * PyArrayObject, a new one will be created and the new object test_modifier + * PyArrayObject, a new one will be created and the new object flag * will be set. */ PyArrayObject* obj_to_array_fortran_allow_conversion(PyObject* input, @@ -524,7 +524,7 @@ /* Require the given PyArrayObject to to be Fortran ordered. If the * the PyArrayObject is already Fortran ordered, do nothing. Else, - * set the Fortran ordering test_modifier and recompute the strides. + * set the Fortran ordering flag and recompute the strides. */ int require_fortran(PyArrayObject* ary) { @@ -533,7 +533,7 @@ int i; npy_intp * strides = array_strides(ary); if (array_is_fortran(ary)) return success; - /* Set the Fortran ordered test_modifier */ + /* Set the Fortran ordered flag */ array_enableflags(ary,NPY_ARRAY_FARRAY); /* Recompute the strides */ strides[0] = strides[nd-1]; diff --git a/src/ext/swig/comm.i b/src/ext/swig/comm.i index baec17fb1..e5ba18307 100644 --- a/src/ext/swig/comm.i +++ b/src/ext/swig/comm.i @@ -28,6 +28,11 @@ using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; %} +%pragma(java) jniclasscode=%{ + static { + System.loadLibrary("interop_comm"); + } +%} // This imports the metrics WRAP_METRICS(IMPORT_METRIC_WRAPPER) @@ -36,10 +41,12 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %{ #include "interop/io/metric_file_stream.h" +#include "interop/io/paths.h" %} %include "interop/io/metric_file_stream.h" +%include "interop/io/paths.h" %define WRAP_METRIC_IO(metric_t) @@ -47,6 +54,7 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %template(write_interop_to_buffer ) illumina::interop::io::write_interop_to_buffer< metric_base::metric_set >; %template(read_interop_from_buffer ) illumina::interop::io::read_interop_from_buffer< metric_base::metric_set >; %template(read_interop ) illumina::interop::io::read_interop< metric_base::metric_set >; + %template(write_interop ) illumina::interop::io::write_interop< metric_base::metric_set >; %enddef diff --git a/src/ext/swig/exceptions/exceptions_impl.i b/src/ext/swig/exceptions/exceptions_impl.i index f061bebd6..816aa408a 100644 --- a/src/ext/swig/exceptions/exceptions_impl.i +++ b/src/ext/swig/exceptions/exceptions_impl.i @@ -51,11 +51,20 @@ namespace std { #elif defined(SWIGPYTHON) +%pythoncode %{ + class exceptionMetaclass(type): + + def __new__(self, name, bases, classdict): + if Exception not in bases: + return type(name, (Exception, )+bases, classdict) + return type(name, bases, classdict) +%} %define WRAP_EXCEPTION_IMPORT(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_CSHARP) %enddef %define WRAP_EXCEPTION(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_CSHARP) %extend NAMESPACE EXCEPTION_CPLUS_PLUS { + %pythoncode { __metaclass__ = exceptionMetaclass } std::string __str__()const{return self->what();} } %enddef diff --git a/src/ext/swig/extends/extends_csharp.i b/src/ext/swig/extends/extends_csharp.i deleted file mode 100644 index 2b53da871..000000000 --- a/src/ext/swig/extends/extends_csharp.i +++ /dev/null @@ -1,242 +0,0 @@ - -%define SHARED_METRIC_SET_METHODS(metric_t) - private long _dataSourceLength; - private bool _dataSourceExists; - public long DataSourceLength { get{return size();} set{} } - public bool DataSourceExists { get{return size() > 0;} set{} } - public byte Version { get{ return (byte)version(); } } - public global::System.Int64 GetKey(int lane, int tile) - { - return (global::System.Int64)metric_t.create_id((ulong)lane, (ulong)tile); - } - public bool HasData { get { return size() > 0; } } - public metric_t GetMetric(long key) - { - try{ - return get_metric((ulong)key); - } - catch(global::System.IndexOutOfRangeException){return null;} - catch(index_out_of_bounds_exception){return null;} - } - public metric_t GetMetric(int lane, int tile) - { - try{ - return get_metric((uint)lane, (uint)tile); - } - catch(global::System.IndexOutOfRangeException){return null;} - catch(index_out_of_bounds_exception){return null;} - } - public metric_t GetMetric(ulong key) - { - try{ - return get_metric(key); - } - catch(global::System.IndexOutOfRangeException){return null;} - catch(index_out_of_bounds_exception){return null;} - } - public global::System.Collections.Generic.IEnumerable< metric_t > GetMetricsInLane(int lane) - { - if(size() == 0) return new global::System.Collections.Generic.List(); - return global::System.Linq.Enumerable.Where(metrics(), m => m.lane() == lane); - } - public global::System.Collections.Generic.List GetTiles(int lane) - { - if(size() == 0) return new global::System.Collections.Generic.List(); - return global::System.Linq.Enumerable.ToList(tile_numbers_for_lane((uint)lane)); - } - public global::System.Collections.Generic.IEnumerable< metric_t > GetMetricsForTile(int lane, int tile) - { - if(size() == 0) return new global::System.Collections.Generic.List(); - return global::System.Linq.Enumerable.Where(metrics(), m => m.lane() == lane && m.tile() == tile); - } - /* - public global::System.Collections.Generic.IEnumerable< metric_t > AllMetrics { get { return metrics(); } } - public global::System.Collections.Generic.IEnumerable AllKeys { get { return keys(); } } - */ - public global::System.Collections.Generic.IEnumerable< metric_t > AllMetrics - { - get - { - global::System.Collections.Generic.Dictionary Lookup = new global::System.Collections.Generic.Dictionary(); - foreach(metric_t metric in metrics()) - { - try{ - Lookup.Add((global::System.Int64)metric.id(), metric); - }catch(global::System.Exception ex) - { - global::System.Console.WriteLine(metric.Lane+"_"+metric.Tile+" == "+Lookup[(global::System.Int64)metric.id()].Lane+"_"+Lookup[(global::System.Int64)metric.id()].Tile); - throw ex; - } - } - return Lookup.Values; - } - } - public global::System.Collections.Generic.IEnumerable AllKeys - { - get - { - global::System.Collections.Generic.Dictionary Lookup = new global::System.Collections.Generic.Dictionary(); - foreach(metric_t metric in metrics()) - { - Lookup.Add((global::System.UInt64)metric.id(), metric); - } - return Lookup.Keys; - } - } - - private run_metrics _runMetricsRef; - internal void AddReference(run_metrics metrics) - { - _runMetricsRef = metrics; - } -%enddef - -%define SHARED_CYCLE_METRIC_SET_METHODS(metric_t) - - - public int MaxCycle { get{ return (int)max_cycle(); } } - public global::System.Int64 GetKey(int lane, int tile, int cycle) - { - return (global::System.Int64)metric_t.create_id((ulong)lane, (ulong)tile, (ulong)cycle); - } - public metric_t GetMetric(int lane, int tile, int cycle) - { - try{ - return get_metric((uint)lane, (uint)tile, (uint)cycle); - } - catch(global::System.IndexOutOfRangeException){return null;} - catch(index_out_of_bounds_exception){return null;} - } - public global::System.Collections.Generic.IEnumerable< metric_t > GetMetricsByCycle(global::System.Collections.Generic.IEnumerable cycles) - { - if(size() == 0) return new global::System.Collections.Generic.List(); - return global::System.Linq.Enumerable.Where(metrics(), m => global::System.Linq.Enumerable.Contains(cycles, (int)m.cycle())); - } - public global::System.Collections.Generic.IEnumerable< metric_t> GetMetricsByCycle(int lane, int tile, global::System.Collections.Generic.IEnumerable cycles) - { - if(size() == 0) return new global::System.Collections.Generic.List(); - global::System.Collections.Generic.List< metric_t> results = new global::System.Collections.Generic.List< metric_t>(); - foreach (int cycle in cycles) - { - metric_t metric = GetMetric(lane, tile, cycle); - if (metric != null) results.Add(metric); - } - return results; - } -%enddef - -%define EXTEND_METRIC_SET(metric_t) - using namespace illumina::interop::model::metrics; - namespace metric_base = illumina::interop::model::metric_base; - - %typemap(cscode) illumina::interop::model::metric_base::metric_set %{ - SHARED_METRIC_SET_METHODS(metric_t) - //SHARED_CYCLE_METRIC_SET_METHODS(metric_t) - %} -%enddef - - -%define EXTEND_CYCLE_METRIC_SET(metric_t) - - using namespace illumina::interop::model::metrics; - namespace metric_base = illumina::interop::model::metric_base; - - %typemap(cscode) illumina::interop::model::metric_base::metric_set %{ - SHARED_METRIC_SET_METHODS(metric_t) - SHARED_CYCLE_METRIC_SET_METHODS(metric_t) - %} -%enddef - -%define EXTEND_Q_METRIC(metric_t) - %typemap(cscode) illumina::interop::model::metric_base::metric_set %{ - SHARED_METRIC_SET_METHODS(metric_t) - SHARED_CYCLE_METRIC_SET_METHODS(metric_t) - public class QScoreBin - { - private byte _lower; - private byte _upper; - private byte _value; - public byte Lower { get{return _lower;} set{_lower=value;} } - public byte Upper { get{return _upper;} set{_upper=value;} } - public byte Value { get{return _value;} set{_value=value;} } - public QScoreBin(){} - public QScoreBin(byte lower, byte upper, byte value) - { - Lower = lower; - Upper = upper; - Value = value; - } - }; - private global::System.Collections.Generic.List _qScoreBins; - public global::System.Collections.Generic.List QScoreBins { get{return _qScoreBins;} set{_qScoreBins=value;} } - - public bool IsBinned { get { return binCount() > 0; }} - public int NumQVals(){ return (int)c_csharp_metrics.count_q_metric_bins(this); } - public bool IsCompressed { get { return NumQVals() > 0 && NumQVals() != 50; } } - public int MaxQVal(){ return (int)c_csharp_metrics.max_qval(this); } - - public void build_bins(instrument_type instrument) - { - c_csharp_metrics.populate_legacy_q_score_bins(this, bins(), instrument); - } - public void populateCumulativeDistributions() - { - c_csharp_metrics.populate_cumulative_distribution(this); - } - %} -%enddef - -%define EXTEND_TILE_METRIC(metric_t) - %typemap(cscode) illumina::interop::model::metric_base::metric_set %{ - SHARED_METRIC_SET_METHODS(metric_t) - private int _controlLane; - public int ControlLane { get{return _controlLane;} set{_controlLane=value;} } - public global::System.Collections.Generic.List Tiles { get{return global::System.Linq.Enumerable.ToList(tile_numbers());} } - %} -%enddef - -/* -%typemap(csout, excode=SWIGEXCODE) illumina::interop::model::metric_base::metric_set& tile_metric_set() -{ - System.IntPtr cPtr = $imcall;$excode - $csclassname ret = null; - if (cPtr != System.IntPtr.Zero) { - ret = new $csclassname(cPtr, $owner); - ret.AddReference(this); - } - return ret; -} -*/ - -%typemap(cscode) illumina::interop::model::metrics::tile_metric %{ - public int LatestExtractedCycle; - public int LatestCalledCycle; - public int LatestQScoredCycle; - public int LatestErrorCycle; - public int cycle(){return -1;}// Hack for bad interface above -%} -%typemap(cscode) illumina::interop::model::metrics::index_metric %{ - public int cycle(){return -1;}// Hack for bad interface above -%} - - -%typemap(cscode) illumina::interop::model::metric_base::base_metric_header %{ - public int max_cycle(){return 0;} // Hack for bad interface above -%} - -%typemap(cscode) illumina::interop::model::metric_base::base_read_metric_header %{ - public int max_cycle(){return 0;} // Hack for bad interface above -%} - -%typemap(cscode) illumina::interop::model::metric_base::base_metric %{ - - public int Tile { get { return (int)tile(); } } - public int Lane { get { return (int)lane(); } } - %} - - -%typemap(cscode) illumina::interop::model::metric_base::base_cycle_metric %{ - - public /*new*/ int Cycle { get { return (int)cycle(); } } - %} - diff --git a/src/ext/swig/extends/extends_impl.i b/src/ext/swig/extends/extends_impl.i deleted file mode 100644 index 507055500..000000000 --- a/src/ext/swig/extends/extends_impl.i +++ /dev/null @@ -1,13 +0,0 @@ - -#if defined(SWIGCSHARP) - %include "extends_csharp.i" -#else - %define EXTEND_METRIC_SET(metric_t) - %enddef - %define EXTEND_CYCLE_METRIC_SET(metric_t) - %enddef - %define EXTEND_Q_METRIC(metric_t) - %enddef - %define EXTEND_TILE_METRIC(metric_t) - %enddef -#endif diff --git a/src/ext/swig/metrics.i b/src/ext/swig/metrics.i index 8dda81e7a..59ea89e49 100644 --- a/src/ext/swig/metrics.i +++ b/src/ext/swig/metrics.i @@ -1,12 +1,10 @@ /** Metrics model and logic */ - %include %include %include %include %import "src/ext/swig/exceptions/exceptions_impl.i" -%include "src/ext/swig/extends/extends_impl.i" %include "src/ext/swig/arrays/arrays_impl.i" %include "util/operator_overload.i" @@ -29,6 +27,12 @@ using System.Runtime.InteropServices; using Illumina.InterOp.Run; %} +%pragma(java) jniclasscode=%{ + static { + System.loadLibrary("interop_metrics"); + } +%} + EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) @@ -44,6 +48,7 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %{ #include "interop/util/time.h" +#include "interop/constants/enum_description.h" %} %ignore set_base(const io::layout::base_metric& base); @@ -60,11 +65,15 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) // Define shared macros //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +%apply uint64_t { io::layout::base_metric::id_t }; %define WRAP_TYPES(metric_t) using namespace illumina::interop::model::metrics; namespace metric_base = illumina::interop::model::metric_base; + %ignore illumina::interop::model::metric_base::metric_set::populate_tile_numbers_for_lane; + %ignore illumina::interop::model::metric_base::metric_set::populate_tile_numbers_for_lane_surface; %ignore illumina::interop::model::metric_base::metric_set::offset_map; + //%ignore illumina::interop::model::metric_base::metric_set::offset_map; TODO Ignore functions that take set are argument %apply size_t { std::map< std::size_t, metric_t >::size_type }; %apply uint64_t { metric_base::metric_set::id_t }; @@ -120,12 +129,7 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// %template(index_info_vector) std::vector< illumina::interop::model::metrics::index_info >; -%template(ulong_vector) std::vector< uint64_t >; -%template(ushort_vector) std::vector< uint16_t >; -%template(uint_vector) std::vector< uint32_t >; -%template(float_vector) std::vector< float >; -%template(bool_vector) std::vector< bool >; -%template(uchar_vector) std::vector< uint8_t >; + %template(tile_metric_map) std::map< uint64_t, illumina::interop::model::metric_base::base_metric >; %template(cycle_metric_map) std::map< uint64_t, illumina::interop::model::metric_base::base_cycle_metric >; %template(read_metric_vector) std::vector< illumina::interop::model::metrics::read_metric >; @@ -148,19 +152,6 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) WRAP_METRICS(INCLUDE_METRIC_WRAPPER) WRAP_METRICS(WRAP_TYPES) -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Deprecated -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -EXTEND_CYCLE_METRIC_SET(corrected_intensity_metric) -EXTEND_CYCLE_METRIC_SET(error_metric) -EXTEND_CYCLE_METRIC_SET(extraction_metric) -EXTEND_CYCLE_METRIC_SET(image_metric) -EXTEND_Q_METRIC(q_metric) -EXTEND_TILE_METRIC(tile_metric) -EXTEND_METRIC_SET(index_metric) -EXTEND_Q_METRIC(q_collapsed_metric) -EXTEND_Q_METRIC(q_by_lane_metric) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -174,16 +165,28 @@ WRAP_METRICS(WRAP_METRIC_SET) %{ #include "interop/logic/metric/extraction_metric.h" #include "interop/logic/metric/q_metric.h" -#include "interop/model/run_metrics.h" #include "interop/logic/utils/metric_type_ext.h" #include "interop/logic/utils/metrics_to_load.h" %} %include "interop/logic/metric/extraction_metric.h" %include "interop/logic/metric/q_metric.h" -%include "interop/model/run_metrics.h" %include "interop/logic/utils/metric_type_ext.h" %include "interop/logic/utils/metrics_to_load.h" +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Run Metrics +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +%{ +#include "interop/model/run_metrics.h" +%} +%include "interop/model/run_metrics.h" +%define WRAP_RUN_METRICS(metric_t) + %template(list_ ## metric_t ## _filenames) illumina::interop::model::metrics::run_metrics::list_filenames< metric_t >; + %template(set_ ## metric_t ## _set) illumina::interop::model::metrics::run_metrics::set< illumina::interop::model::metric_base::metric_set< metric_t> >; + %template(metric_t ## _set) illumina::interop::model::metrics::run_metrics::get_metric_set< metric_t >; +%enddef +WRAP_METRICS(WRAP_RUN_METRICS) diff --git a/src/ext/swig/plot.i b/src/ext/swig/plot.i index 5e6eca9e5..d1aa13f0a 100644 --- a/src/ext/swig/plot.i +++ b/src/ext/swig/plot.i @@ -14,6 +14,7 @@ %import "src/ext/swig/run.i" %import "src/ext/swig/metrics.i" + // Ensure all the modules import the shared namespace %pragma(csharp) moduleimports=%{ using Illumina.InterOp.Metrics; @@ -28,6 +29,11 @@ using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; %} +%pragma(java) jniclasscode=%{ + static { + System.loadLibrary("interop_plot"); + } +%} // This imports the metrics WRAP_METRICS(IMPORT_METRIC_WRAPPER) @@ -50,6 +56,9 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) #include "interop/model/plot/heatmap_data.h" #include "interop/model/plot/flowcell_data.h" %} + +%ignore illumina::interop::model::plot::filter_options::option_iterator; + %include "interop/model/plot/axes.h" %include "interop/model/plot/filter_options.h" %include "interop/model/plot/chart_data.h" diff --git a/src/ext/swig/run.i b/src/ext/swig/run.i index 55d4a2700..0fb1970c6 100644 --- a/src/ext/swig/run.i +++ b/src/ext/swig/run.i @@ -7,6 +7,14 @@ %include "src/ext/swig/exceptions/exceptions_impl.i" %include "util/operator_overload.i" +%rename(to_int) operator uint64_t; + +%pragma(java) jniclasscode=%{ + static { + System.loadLibrary("interop_run"); + } +%} + %{ #include "interop/interop.h" #include "interop/model/run/cycle_range.h" @@ -16,17 +24,50 @@ #include "interop/model/run/info.h" #include "interop/model/run/parameters.h" #include "interop/logic/metric/q_metric.h" +#include "interop/logic/utils/enums.h" %} +// Primitive array types %template(string_vector) std::vector< std::string >; +%template(ulong_vector) std::vector< uint64_t >; +%template(ushort_vector) std::vector< uint16_t >; +%template(uint_vector) std::vector< uint32_t >; +%template(float_vector) std::vector< float >; +%template(bool_vector) std::vector< bool >; +%template(uchar_vector) std::vector< uint8_t >; + +#if !defined(SWIGPYTHON) +%template(size_vector) std::vector< size_t >; +#endif %include "interop/interop.h" %include "interop/constants/enums.h" +%include "interop/constants/enum_description.h" %include "interop/model/run/cycle_range.h" %include "interop/model/run/read_info.h" %include "interop/model/run/flowcell_layout.h" %include "interop/model/run/image_dimensions.h" %include "interop/model/run/info.h" %include "interop/model/run/parameters.h" +%include "interop/logic/utils/enums.h" %template(read_info_vector) std::vector; + + + +%define WRAP_ENUM(ENUM) +%template(list_##ENUM) illumina::interop::constants::list_enum_names< illumina::interop::constants:: ENUM >; +%enddef + +WRAP_ENUM(metric_type) +WRAP_ENUM(metric_group) +WRAP_ENUM(tile_naming_method) +WRAP_ENUM(dna_bases) +WRAP_ENUM(surface_type) +WRAP_ENUM(instrument_type) +WRAP_ENUM(metric_base_type) +WRAP_ENUM(plot_colors) +WRAP_ENUM(bar_plot_options) +WRAP_ENUM(metric_data) +WRAP_ENUM(metric_feature_type) + diff --git a/src/ext/swig/summary.i b/src/ext/swig/summary.i index 656f343d0..e4fc9316e 100644 --- a/src/ext/swig/summary.i +++ b/src/ext/swig/summary.i @@ -27,6 +27,11 @@ using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; %} +%pragma(java) jniclasscode=%{ + static { + System.loadLibrary("interop_summary"); + } +%} // This imports the metrics WRAP_METRICS(IMPORT_METRIC_WRAPPER) @@ -65,11 +70,15 @@ WRAP_VECTOR(illumina::interop::model::summary::run_summary); // // Setup typemaps for summary metrics // + +%ignore illumina::interop::model::summary::lane_summary::at(const size_type) const; +%ignore illumina::interop::model::summary::read_summary::at(const size_type) const; WRAP_AS_VECTOR(illumina::interop::model::summary::surface_summary); WRAP_AS_VECTOR(illumina::interop::model::summary::lane_summary); WRAP_AS_VECTOR(illumina::interop::model::summary::read_summary); -%template(lane_summary_vector) std::vector; + +%template(surface_summary_vector) std::vector; %template(lane_summary_vector) std::vector; %template(read_summary_vector) std::vector; diff --git a/src/ext/swig/imaging.i b/src/ext/swig/table.i similarity index 90% rename from src/ext/swig/imaging.i rename to src/ext/swig/table.i index 1be0305eb..84368cb85 100644 --- a/src/ext/swig/imaging.i +++ b/src/ext/swig/table.i @@ -4,6 +4,7 @@ %include %include %include +%include %include "util/operator_overload.i" ////////////////////////////////////////////// @@ -28,17 +29,18 @@ using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; %} +%pragma(java) jniclasscode=%{ + static { + System.loadLibrary("interop_table"); + } +%} // This imports the metrics WRAP_METRICS(IMPORT_METRIC_WRAPPER) // This allows exceptions to be imported, but not belong to the module EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) -#if defined(HAVE_UINT64_SIZE_T) && defined(SWIGPYTHON) // Python workaround - %template(map_id_offset) std::map; -#else - %template(map_id_offset) std::map; -#endif +%template(map_id_offset) std::map; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Imaging model diff --git a/src/interop/CMakeLists.txt b/src/interop/CMakeLists.txt index 588a3b35e..d9d9a6df0 100644 --- a/src/interop/CMakeLists.txt +++ b/src/interop/CMakeLists.txt @@ -26,9 +26,12 @@ set(SRCS logic/table/create_imaging_table.cpp util/time.cpp util/filesystem.cpp - logic/utils/metrics_to_load.cpp) + logic/utils/metrics_to_load.cpp + model/summary/index_summary.cpp + ) set(HEADERS + ../../interop/io/paths.h ../../interop/model/summary/cycle_state_summary.h ../../interop/model/summary/lane_summary.h ../../interop/model/summary/metric_stat.h @@ -141,7 +144,13 @@ set(HEADERS ../../interop/logic/utils/metrics_to_load.h ../../interop/io/format/default_layout.h ../../interop/io/format/stream_membuf.h - ../../interop/model/summary/surface_summary.h ../../interop/model/summary/stat_summary.h) + ../../interop/model/summary/surface_summary.h + ../../interop/model/summary/stat_summary.h + ../../interop/util/indirect_range_iterator.h + ../../interop/util/map.h + ../../interop/util/timer.h + ../../interop/constants/enum_description.h + ) set(INTEROP_HEADERS ${HEADERS} PARENT_SCOPE) if(COMPILER_IS_GNUCC_OR_CLANG) @@ -158,8 +167,16 @@ elseif(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS) endif() -add_library(${INTEROP_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS}) -add_library(${INTEROP_DL_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS} ) +# Shared libraries on windows are currently not supported, but we may in the future. +set(SWIG_VERSION_INFO "") +if(MSVC AND "${LIBRARY_TYPE}" STREQUAL "SHARED") + set(SWIG_VERSION_INFO ${CMAKE_CURRENT_BINARY_DIR}/interop_lib_version.rc) + set(LIB_NAME "Core C++ InterOp Library") # Used to configure the version info file + configure_file(${CMAKE_SOURCE_DIR}/cmake/version.rc.in ${SWIG_VERSION_INFO} @ONLY) # Requires: LIB_NAME, VERSION_LIST and VERSION +endif() + +add_library(${INTEROP_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS} ${SWIG_VERSION_INFO}) +add_library(${INTEROP_DL_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS} ${SWIG_VERSION_INFO} ) add_dependencies(${INTEROP_LIB} version) add_dependencies(${INTEROP_DL_LIB} version) diff --git a/src/interop/logic/metric/extraction_metric.cpp b/src/interop/logic/metric/extraction_metric.cpp index 4837ff6bf..1451048c6 100644 --- a/src/interop/logic/metric/extraction_metric.cpp +++ b/src/interop/logic/metric/extraction_metric.cpp @@ -17,9 +17,9 @@ namespace illumina { namespace interop { namespace logic { namespace metric { typedef model::metric_base::metric_set::const_iterator const_iterator; if(metrics.size()==0)return; - if(n < metrics.size()) throw model::invalid_parameter("Buffer size too small for metric set"); - if(channel >= metrics.metrics()[0].channel_count()) - throw model::invalid_parameter("Channel exceeds channel count"); + if(n < metrics.size()) INTEROP_THROW(model::invalid_parameter, "Buffer size too small for metric set"); + if(channel >= metrics.at(0).channel_count()) + INTEROP_THROW(model::invalid_parameter, "Channel exceeds channel count"); for (const_iterator it = metrics.begin(); it != metrics.end(); ++it, ++focus_scores) *focus_scores = it->focus_score(channel); } diff --git a/src/interop/logic/metric/q_metric.cpp b/src/interop/logic/metric/q_metric.cpp index d6ae06a6f..f0ea7e994 100644 --- a/src/interop/logic/metric/q_metric.cpp +++ b/src/interop/logic/metric/q_metric.cpp @@ -117,7 +117,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { // 0 is a sentinel that indicates legacy binning is not required if(q_metric_set.version() > 4) return 0; // Version 5 and later do not require legacy binning - if(!q_metric_set.bins().empty()) return 0; // If the metrics already have a header they do not require binning + if(!q_metric_set.get_bins().empty()) return 0; // If the metrics already have a header they do not require binning const size_t max_bin_count = 7; model::metric_base::metric_set::const_iterator beg = q_metric_set.begin(), @@ -222,13 +222,13 @@ namespace illumina { namespace interop { namespace logic { namespace metric const uint_t q20_idx = static_cast(index_for_q_value(metric_set, 20)); const uint_t q30_idx = static_cast(index_for_q_value(metric_set, 30)); - collapsed.set_version(metric_set.version()); + collapsed.set_version(model::metrics::q_collapsed_metric::LATEST_VERSION); for(const_iterator beg = metric_set.begin(), end = metric_set.end();beg != end;++beg) { const uint_t q20 = beg->total_over_qscore(q20_idx); const uint_t q30 = beg->total_over_qscore(q30_idx); const uint_t total = beg->sum_qscore(); - const uint_t median = beg->median(metric_set.bins()); + const uint_t median = beg->median(metric_set.get_bins()); collapsed.insert(model::metrics::q_collapsed_metric(beg->lane(), beg->tile(), beg->cycle(), @@ -254,7 +254,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric typedef model::metric_base::base_cycle_metric::id_t id_t; bylane = static_cast(metric_set); - bylane.set_version(metric_set.version()); + bylane.set_version(model::metrics::q_by_lane_metric::LATEST_VERSION); for(const_iterator beg = metric_set.begin(), end = metric_set.end();beg != end;++beg) { const id_t id = model::metric_base::base_cycle_metric::create_id(beg->lane(), 0, beg->cycle()); diff --git a/src/interop/logic/plot/plot_by_cycle.cpp b/src/interop/logic/plot/plot_by_cycle.cpp index 05a6e3f67..4736f7e93 100644 --- a/src/interop/logic/plot/plot_by_cycle.cpp +++ b/src/interop/logic/plot/plot_by_cycle.cpp @@ -147,13 +147,14 @@ namespace illumina { namespace interop { namespace logic { namespace plot model::plot::plot_data& data) { data.clear(); + if(metrics.is_group_empty(logic::utils::to_group(type))) return; if(!options.all_cycles()) - INTEROP_THROW(model::invalid_filter_option, "Filtering by cycle is not supported"); + INTEROP_THROW(model::invalid_filter_option, "Filtering by cycle is not supported");// TODO: Remove this! if(!options.all_reads()) - INTEROP_THROW(model::invalid_filter_option, "Filtering by read is not supported"); + INTEROP_THROW(model::invalid_filter_option, "Filtering by read is not supported");// TODO: Remove this! if(!utils::is_cycle_metric(type)) INTEROP_THROW(model::invalid_filter_option, "Only cycle metrics are supported"); - options.validate(type, metrics.run_info()); + options.validate(type, metrics.run_info()); // TODO: Check ignored? size_t max_cycle=0; switch(logic::utils::to_group(type)) { @@ -166,7 +167,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { metric::metric_value proxy(i); max_cycle = populate_metric_average_by_cycle( - metrics.get_set(), + metrics.get(), proxy, options, type, @@ -180,7 +181,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot const size_t channel = options.channel(); metric::metric_value proxy(channel); max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), + metrics.get(), proxy, options, type, @@ -198,7 +199,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot metric::metric_value proxy( static_cast(i)); max_cycle = populate_metric_average_by_cycle( - metrics.get_set(), + metrics.get(), proxy, options, type, @@ -212,7 +213,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot const constants::dna_bases base = options.dna_base(); metric::metric_value proxy(base); max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), + metrics.get(), proxy, options, type, @@ -226,11 +227,11 @@ namespace illumina { namespace interop { namespace logic { namespace plot typedef model::metrics::q_collapsed_metric metric_t; metric::metric_value proxy2; - if(0 == metrics.get_set().size()) - logic::metric::create_collapse_q_metrics(metrics.get_set(), - metrics.get_set()); + if(0 == metrics.get().size()) + logic::metric::create_collapse_q_metrics(metrics.get(), + metrics.get()); max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), + metrics.get(), proxy2, options, type, @@ -242,7 +243,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot data.assign(1, model::plot::series()); metric::metric_value proxy3; max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), + metrics.get(), proxy3, options, type, diff --git a/src/interop/logic/plot/plot_by_lane.cpp b/src/interop/logic/plot/plot_by_lane.cpp index 905834ab6..630fb306f 100644 --- a/src/interop/logic/plot/plot_by_lane.cpp +++ b/src/interop/logic/plot/plot_by_lane.cpp @@ -73,12 +73,14 @@ namespace illumina { namespace interop { namespace logic { namespace plot model::invalid_metric_type, model::invalid_filter_option) { + data.clear(); + if(metrics.is_group_empty(logic::utils::to_group(type))) return; if(utils::is_cycle_metric(type)) - INTEROP_THROW(model::invalid_metric_type, "Cycle metrics are unsupported"); + INTEROP_THROW(model::invalid_metric_type, "Cycle metrics are unsupported: " << constants::to_string(type)); options.validate(type, metrics.run_info()); data.assign(1, model::plot::series(utils::to_description(type), "Blue")); metric::metric_value proxy3(options.read()); - populate_candle_stick_by_lane(metrics.get_set(), proxy3, options, type, + populate_candle_stick_by_lane(metrics.get(), proxy3, options, type, data[0]); const size_t read_count = metrics.run_info().reads().size(); @@ -91,7 +93,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot const constants::metric_type second_type = (type==constants::Clusters ? constants::ClustersPF : constants::ClusterCountPF); //(type==constants::Density ? constants::DensityPF : constants::ClusterCountPF); - populate_candle_stick_by_lane(metrics.get_set(), proxy3, options, second_type, + populate_candle_stick_by_lane(metrics.get(), proxy3, options, second_type, data[1]); } diff --git a/src/interop/logic/plot/plot_flowcell_map.cpp b/src/interop/logic/plot/plot_flowcell_map.cpp index f86065175..f6f32ee9e 100644 --- a/src/interop/logic/plot/plot_flowcell_map.cpp +++ b/src/interop/logic/plot/plot_flowcell_map.cpp @@ -74,8 +74,11 @@ namespace illumina { namespace interop { namespace logic { namespace plot model::invalid_metric_type, model::index_out_of_bounds_exception) { - const model::run::flowcell_layout& layout = metrics.run_info().flowcell(); data.clear(); + if(metrics.is_group_empty(logic::utils::to_group(type))) return; + options.validate(type, metrics.run_info()); + + const model::run::flowcell_layout& layout = metrics.run_info().flowcell(); if(buffer == 0 || tile_buffer==0) data.resize(layout.lane_count(), layout.total_swaths(layout.surface_count() > 1 && !options.is_specific_surface()), @@ -89,7 +92,6 @@ namespace illumina { namespace interop { namespace logic { namespace plot std::vector values_for_scaling; values_for_scaling.reserve(data.length()); - options.validate(type, metrics.run_info()); if(utils::is_cycle_metric(type) && options.all_cycles()) INTEROP_THROW( model::invalid_filter_option, "All cycles is unsupported"); @@ -101,7 +103,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { typedef model::metrics::tile_metric metric_t; typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); + const metric_set_t& metric_set = metrics.get(); metric::metric_value proxy(options.read()); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); @@ -111,7 +113,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { typedef model::metrics::extraction_metric metric_t; typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); + const metric_set_t& metric_set = metrics.get(); const size_t channel = options.channel(); if(options.all_channels(type)) INTEROP_THROW(model::invalid_filter_option, "All channels is unsupported"); @@ -124,7 +126,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { typedef model::metrics::corrected_intensity_metric metric_t; typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); + const metric_set_t& metric_set = metrics.get(); const constants::dna_bases base = options.dna_base(); if(options.all_bases(type)) INTEROP_THROW( model::invalid_filter_option, "All bases is unsupported"); @@ -137,10 +139,10 @@ namespace illumina { namespace interop { namespace logic { namespace plot { typedef model::metrics::q_collapsed_metric metric_t; typedef model::metric_base::metric_set metric_set_t; - metric_set_t &metric_set = metrics.get_set(); + metric_set_t &metric_set = metrics.get(); if(0 == metric_set.size()) { - logic::metric::create_collapse_q_metrics(metrics.get_set(), metric_set); + logic::metric::create_collapse_q_metrics(metrics.get(), metric_set); if(type == constants::AccumPercentQ20 || type == constants::AccumPercentQ30) logic::metric::populate_cumulative_distribution(metric_set); } @@ -153,7 +155,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { typedef model::metrics::error_metric metric_t; typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); + const metric_set_t& metric_set = metrics.get(); metric::metric_value proxy; populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); diff --git a/src/interop/logic/plot/plot_qscore_heatmap.cpp b/src/interop/logic/plot/plot_qscore_heatmap.cpp index 92198fe45..fef51e668 100644 --- a/src/interop/logic/plot/plot_qscore_heatmap.cpp +++ b/src/interop/logic/plot/plot_qscore_heatmap.cpp @@ -113,12 +113,12 @@ namespace illumina { namespace interop { namespace logic { namespace plot INTEROP_ASSERTMSG(data.column_count() > 0, data.column_count() << ", " << metric_set.size() << ", " << metric_set.bin_count() << ", " << metric::is_compressed(metric_set) << ", " - << metric_set.bins().back().upper()); + << metric_set.get_bins().back().upper()); const bool is_compressed = logic::metric::is_compressed(metric_set); if(is_compressed) populate_heatmap_from_compressed(metric_set.begin(), metric_set.end(), - metric_set.bins(), + metric_set.get_bins(), options, data); else @@ -127,8 +127,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot options, data); normalize_heatmap(data); - remap_to_bins(metric_set.bins().begin(), - metric_set.bins().end(), + remap_to_bins(metric_set.get_bins().begin(), + metric_set.get_bins().end(), data.row_count(), data); } @@ -143,26 +143,28 @@ namespace illumina { namespace interop { namespace logic { namespace plot void plot_qscore_heatmap(model::metrics::run_metrics& metrics, const model::plot::filter_options& options, model::plot::heatmap_data& data, - float* buffer) + float* buffer, + const size_t) throw(model::index_out_of_bounds_exception, model::invalid_filter_option) { - options.validate(constants::QScore, metrics.run_info()); data.clear(); if(options.is_specific_surface()) { typedef model::metrics::q_metric metric_t; - if (metrics.get_set().size() == 0)return; - populate_heatmap(metrics.get_set(), options, data, buffer); + if (metrics.get().size() == 0)return; + options.validate(constants::QScore, metrics.run_info()); + populate_heatmap(metrics.get(), options, data, buffer); } else { typedef model::metrics::q_by_lane_metric metric_t; - if(0 == metrics.get_set().size()) - logic::metric::create_q_metrics_by_lane(metrics.get_set(), - metrics.get_set()); - if (metrics.get_set().size() == 0)return; - populate_heatmap(metrics.get_set(), options, data, buffer); + if(0 == metrics.get().size()) + logic::metric::create_q_metrics_by_lane(metrics.get(), + metrics.get()); + if (metrics.get().size() == 0)return; + options.validate(constants::QScore, metrics.run_info()); + populate_heatmap(metrics.get(), options, data, buffer); } data.set_xrange(0, static_cast(data.row_count())); @@ -185,7 +187,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot */ size_t count_rows_for_heatmap(const model::metrics::run_metrics& metrics) { - return metrics.get_set< model::metrics::q_metric >().max_cycle(); + return metrics.get< model::metrics::q_metric >().max_cycle(); } /** Count number of columns for the heat map * @@ -194,7 +196,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot */ size_t count_columns_for_heatmap(const model::metrics::run_metrics& metrics) { - return logic::metric::max_qval(metrics.get_set< model::metrics::q_metric >()); + return logic::metric::max_qval(metrics.get< model::metrics::q_metric >()); } diff --git a/src/interop/logic/plot/plot_qscore_histogram.cpp b/src/interop/logic/plot/plot_qscore_histogram.cpp index 42d0b6df8..6a584966b 100644 --- a/src/interop/logic/plot/plot_qscore_histogram.cpp +++ b/src/interop/logic/plot/plot_qscore_histogram.cpp @@ -153,8 +153,17 @@ namespace illumina { namespace interop { namespace logic { namespace plot model::invalid_filter_option) { typedef model::plot::bar_point Point; - options.validate(constants::QScore, metrics.run_info()); data.clear(); + if(options.is_specific_surface()) + { + if(metrics.get< model::metrics::q_metric >().empty())return; + } + else + { + if(metrics.get< model::metrics::q_metric >().empty() && + metrics.get< model::metrics::q_by_lane_metric >().empty() )return; + } + options.validate(constants::QScore, metrics.run_info()); const size_t first_cycle = options.all_reads() ? 1 : metrics.run_info().read(options.read()).first_cycle(); data.push_back(model::plot::series("Q Score", "Blue", model::plot::series::Bar)); @@ -178,23 +187,19 @@ namespace illumina { namespace interop { namespace logic { namespace plot typedef model::metrics::q_metric metric_t; const size_t last_cycle = get_last_filtered_cycle(metrics.run_info(), options, - metrics.get_set().max_cycle()); - if(metrics.get_set().size() == 0) - { - //data.clear(); // TODO: Add this back? - return; - } + metrics.get().max_cycle()); + if(metrics.get().size() == 0) return; populate_distribution( - metrics.get_set().begin(), - metrics.get_set().end(), + metrics.get().begin(), + metrics.get().end(), options, first_cycle, last_cycle, histogram); axis_scale = scale_histogram(histogram); - if(!metrics.get_set().bins().empty()) - max_x_value=plot_binned_histogram(metrics.get_set().bins().begin(), - metrics.get_set().bins().end(), + if(!metrics.get().bins().empty()) + max_x_value=plot_binned_histogram(metrics.get().bins().begin(), + metrics.get().bins().end(), histogram, data[0]); else max_x_value=plot_unbinned_histogram(histogram, data[0]); @@ -202,29 +207,25 @@ namespace illumina { namespace interop { namespace logic { namespace plot else { typedef model::metrics::q_by_lane_metric metric_t; - if(0 == metrics.get_set().size()) - logic::metric::create_q_metrics_by_lane(metrics.get_set(), - metrics.get_set()); - if(0 == metrics.get_set().size()) - { - //data.clear(); // TODO: Add this back? - return; - } + if(0 == metrics.get().size()) + logic::metric::create_q_metrics_by_lane(metrics.get(), + metrics.get()); + if(0 == metrics.get().size()) return; const size_t last_cycle = get_last_filtered_cycle(metrics.run_info(), options, - metrics.get_set().max_cycle()); - INTEROP_ASSERT(0 != metrics.get_set().size()); + metrics.get().max_cycle()); + INTEROP_ASSERT(0 != metrics.get().size()); populate_distribution( - metrics.get_set().begin(), - metrics.get_set().end(), + metrics.get().begin(), + metrics.get().end(), options, first_cycle, last_cycle, histogram); axis_scale = scale_histogram(histogram); - if(!metrics.get_set().bins().empty()) - max_x_value=plot_binned_histogram(metrics.get_set().bins().begin(), - metrics.get_set().bins().end(), + if(!metrics.get().bins().empty()) + max_x_value=plot_binned_histogram(metrics.get().bins().begin(), + metrics.get().bins().end(), histogram, data[0]); else max_x_value=plot_unbinned_histogram(histogram, data[0]); diff --git a/src/interop/logic/plot/plot_sample_qc.cpp b/src/interop/logic/plot/plot_sample_qc.cpp index ed7a5ff57..f0a86cb40 100644 --- a/src/interop/logic/plot/plot_sample_qc.cpp +++ b/src/interop/logic/plot/plot_sample_qc.cpp @@ -7,8 +7,8 @@ */ #include "interop/logic/plot/plot_sample_qc.h" -#include "interop/util/math.h" #include "interop/logic/utils/enums.h" +#include "interop/logic/utils/metric_type_ext.h" namespace illumina { namespace interop { namespace logic { namespace plot { @@ -26,25 +26,25 @@ namespace illumina { namespace interop { namespace logic { namespace plot { model::plot::data_point_collection& points) { typedef model::metric_base::metric_set index_metric_set_t; - typedef std::map index_count_map_t; + typedef INTEROP_UNORDERED_MAP(std::string, ::uint64_t) index_count_map_t; typedef typename index_count_map_t::iterator map_iterator; typedef typename index_count_map_t::const_iterator const_map_iterator; typedef typename model::metrics::index_metric::const_iterator const_index_iterator; index_count_map_t index_count_map; - float pf_cluster_count_total = 0; + ::uint64_t pf_cluster_count_total = 0; for(typename index_metric_set_t::const_iterator b = index_metrics.begin(), e = index_metrics.end();b != e;++b) { if(lane != b->lane()) continue; try { const model::metrics::tile_metric &tile_metric = tile_metrics.get_metric(b->lane(), b->tile()); - pf_cluster_count_total += tile_metric.cluster_count_pf(); + pf_cluster_count_total += static_cast< ::uint64_t >( tile_metric.cluster_count_pf()); for(const_index_iterator ib = b->indices().begin(), ie = b->indices().end();ib != ie;++ib) { map_iterator found_index = index_count_map.find(ib->index_seq()); - if(found_index == index_count_map.end()) index_count_map[ib->index_seq()] = ib->count(); - else found_index->second += ib->count(); + if(found_index == index_count_map.end()) index_count_map[ib->index_seq()] = ib->cluster_count(); + else found_index->second += ib->cluster_count(); } } catch(const model::index_out_of_bounds_exception&){continue;} // TODO: check better? @@ -61,7 +61,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot { std::stable_sort(keys.begin(), keys.end()); for(typename std::vector::const_iterator b = keys.begin(), e = keys.end();b != e;++b,++i) { - const float height = (pf_cluster_count_total==0) ? 0 : index_count_map[*b] / pf_cluster_count_total * 100.0f; + const float height = (pf_cluster_count_total==0) ? 0.0f : 100.0f * index_count_map[*b] / pf_cluster_count_total; points[i].set(i+1.0f, height, 1.0f); max_height = std::max(max_height, height); } @@ -81,12 +81,15 @@ namespace illumina { namespace interop { namespace logic { namespace plot { throw(model::index_out_of_bounds_exception) { typedef model::plot::series bar_series_t; + data.clear(); + if(metrics.is_group_empty(constants::Tile) || + metrics.is_group_empty(constants::Index)) return; data.set_xlabel("Index Number"); data.set_ylabel("% Reads Identified (PF)"); data.assign(1, bar_series_t("% reads", "Green", bar_series_t::Bar)); data[0].add_option(constants::to_string(constants::Centered)); - if(metrics.get_set().size() == 0) + if(metrics.get().size() == 0) { if(metrics.run_info().is_indexed()) { @@ -100,8 +103,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot { return; } - const float max_height = populate_reads_identified(metrics.get_set(), - metrics.get_set(), + const float max_height = populate_reads_identified(metrics.get(), + metrics.get(), lane, data[0]); auto_scale(data); diff --git a/src/interop/logic/summary/index_summary.cpp b/src/interop/logic/summary/index_summary.cpp index b91d594b6..c381b1d89 100644 --- a/src/interop/logic/summary/index_summary.cpp +++ b/src/interop/logic/summary/index_summary.cpp @@ -22,20 +22,21 @@ namespace illumina { namespace interop { namespace logic { namespace summary { */ template void summarize_index_metrics(I beg, - I end, - const model::metric_base::metric_set& tile_metrics, - const size_t lane, - model::summary::index_lane_summary &summary) + I end, + const model::metric_base::metric_set& tile_metrics, + const size_t lane, + model::summary::index_lane_summary &summary) throw(model::index_out_of_bounds_exception) { typedef typename model::summary::index_lane_summary::read_count_t read_count_t; typedef typename model::metrics::index_metric::const_iterator const_index_iterator; typedef model::summary::index_count_summary index_count_summary; - typedef std::map index_count_map_t; + typedef INTEROP_UNORDERED_MAP(std::string, index_count_summary) index_count_map_t; typedef typename index_count_map_t::iterator map_iterator; + if(beg == end || tile_metrics.empty()) return; index_count_map_t index_count_map; - size_t total_mapped_reads = 0; + ::uint64_t total_mapped_reads = 0; read_count_t pf_cluster_count_total = 0; read_count_t cluster_count_total = 0; for(;beg != end;++beg) @@ -57,13 +58,13 @@ namespace illumina { namespace interop { namespace logic { namespace summary { ib->index2(), ib->sample_id(), ib->sample_proj(), - ib->count()); + ib->cluster_count()); } else { - found_index->second += ib->count(); + found_index->second += ib->cluster_count(); } - total_mapped_reads += ib->count(); + total_mapped_reads += ib->cluster_count(); } } catch(const model::index_out_of_bounds_exception&){continue;} // TODO: check better @@ -117,9 +118,9 @@ namespace illumina { namespace interop { namespace logic { namespace summary { model::summary::index_lane_summary &summary) throw(model::index_out_of_bounds_exception) { - summarize_index_metrics(metrics.get_set().begin(), - metrics.get_set().end(), - metrics.get_set(), lane, + summarize_index_metrics(metrics.get().begin(), + metrics.get().end(), + metrics.get(), lane, summary); } /** Summarize a collection index metrics @@ -136,6 +137,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary { model::summary::index_flowcell_summary &summary) throw(model::index_out_of_bounds_exception) { + if(index_metrics.empty() || tile_metrics.empty()) return; summary.resize(lane_count); for(size_t lane=1;lane <= lane_count;++lane) { @@ -154,8 +156,8 @@ namespace illumina { namespace interop { namespace logic { namespace summary { throw(model::index_out_of_bounds_exception) { const size_t lane_count = metrics.run_info().flowcell().lane_count(); - summarize_index_metrics(metrics.get_set(), - metrics.get_set(), + summarize_index_metrics(metrics.get(), + metrics.get(), lane_count, summary); } diff --git a/src/interop/logic/summary/run_summary.cpp b/src/interop/logic/summary/run_summary.cpp index 2e7d090cf..8c5ed9c5f 100644 --- a/src/interop/logic/summary/run_summary.cpp +++ b/src/interop/logic/summary/run_summary.cpp @@ -33,22 +33,31 @@ namespace illumina { namespace interop { namespace logic { namespace summary size_t tile_count_for_lane = 0; for(unsigned int surface=0;surface < surface_count;++surface) { - metrics.get_set().tile_numbers_for_lane_surface(tile_count_set, lane+1, surface+1, naming_convention); - metrics.get_set().tile_numbers_for_lane_surface(tile_count_set, lane+1, surface+1, naming_convention); - metrics.get_set().tile_numbers_for_lane_surface(tile_count_set, lane+1, surface+1, naming_convention); - metrics.get_set().tile_numbers_for_lane_surface(tile_count_set, lane+1, surface+1, naming_convention); + metrics.get().populate_tile_numbers_for_lane_surface(tile_count_set, + lane+1, + surface+1, + naming_convention); + metrics.get().populate_tile_numbers_for_lane_surface(tile_count_set, + lane+1, + surface+1, + naming_convention); + metrics.get().populate_tile_numbers_for_lane_surface(tile_count_set, + lane+1, + surface+1, + naming_convention); + metrics.get().populate_tile_numbers_for_lane_surface(tile_count_set, + lane+1, + surface+1, + naming_convention); if(surface_count > 1) { - for(size_t read = 0; read < summary.size(); ++read) - { + for (size_t read = 0; read < summary.size(); ++read) summary[read][lane][surface].tile_count(tile_count_set.size()); - } } tile_count_for_lane += tile_count_set.size(); tile_count_set.clear(); } - for(size_t read=0;read().begin(), - metrics.get_set().end(), + summarize_tile_metrics(metrics.get().begin(), + metrics.get().end(), naming_method, summary); - summarize_error_metrics(metrics.get_set().begin(), - metrics.get_set().end(), + summarize_error_metrics(metrics.get().begin(), + metrics.get().end(), cycle_to_read, naming_method, summary, skip_median); INTEROP_ASSERT(metrics.run_info().channels().size()>0); const size_t intensity_channel = utils::expected2actual_map(metrics.run_info().channels())[0]; - summarize_extraction_metrics(metrics.get_set().begin(), - metrics.get_set().end(), + summarize_extraction_metrics(metrics.get().begin(), + metrics.get().end(), cycle_to_read, intensity_channel, naming_method, summary, skip_median); - if(0 == metrics.get_set().size()) - logic::metric::create_collapse_q_metrics(metrics.get_set(), - metrics.get_set()); - summarize_collapsed_quality_metrics(metrics.get_set().begin(), - metrics.get_set().end(), + if(0 == metrics.get().size()) + logic::metric::create_collapse_q_metrics(metrics.get(), + metrics.get()); + summarize_collapsed_quality_metrics(metrics.get().begin(), + metrics.get().end(), cycle_to_read, naming_method, summary); summarize_tile_count(metrics, summary); - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), + summarize_cycle_state(metrics.get(), + metrics.get(), cycle_to_read, &model::summary::cycle_state_summary::error_cycle_range, summary); - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), + summarize_cycle_state(metrics.get(), + metrics.get(), cycle_to_read, &model::summary::cycle_state_summary::extracted_cycle_range, summary); - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), + summarize_cycle_state(metrics.get(), + metrics.get(), cycle_to_read, &model::summary::cycle_state_summary::qscored_cycle_range, summary); // Summarize called cycle state - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), + summarize_cycle_state(metrics.get(), + metrics.get(), cycle_to_read, &model::summary::cycle_state_summary::called_cycle_range, summary); - // Remove the empty lane summary entries - size_t max_lane_count = 0; - for(size_t read=0;read + template void populate_imaging_table_data_by_cycle(InputIterator beg, InputIterator end, const size_t q20_idx, @@ -40,15 +40,16 @@ namespace illumina { namespace interop { namespace logic { namespace table const constants::tile_naming_method naming_method, const summary::read_cycle_vector_t& cycle_to_read, const std::vector& columns, - const std::map& row_offset, + const row_offset_map_t& row_offset, const size_t column_count, OutputIterator data_beg, OutputIterator data_end) { + typedef model::metric_base::base_metric::id_t id_t; for(;beg != end;++beg) { - const Id_t id = beg->id(); - typename std::map::const_iterator row_it = row_offset.find(id); + const id_t id = beg->cycle_hash(); + typename row_offset_map_t::const_iterator row_it = row_offset.find(id); const size_t row = row_it->second; const summary::read_cycle& read = cycle_to_read[beg->cycle()-1]; if(data_beg[row*column_count]==0) @@ -69,6 +70,7 @@ namespace illumina { namespace interop { namespace logic { namespace table read, q20_idx, q30_idx, + 0, naming_method, columns, data_beg+row*column_count, @@ -78,6 +80,7 @@ namespace illumina { namespace interop { namespace logic { namespace table read.number, q20_idx, q30_idx, + 0, naming_method, columns, data_beg+row*column_count, data_end); @@ -96,14 +99,14 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param data_beg iterator to start of table data * @param data_end iterator to end of table data */ - template + template void populate_imaging_table_data_by_cycle(const MetricSet& metrics, const size_t q20_idx, const size_t q30_idx, const constants::tile_naming_method naming_method, const summary::read_cycle_vector_t& cycle_to_read, const std::vector& columns, - const std::map& row_offset, + const row_offset_map_t& row_offset, const size_t column_count, OutputIterator data_beg, OutputIterator data_end) { @@ -120,9 +123,9 @@ namespace illumina { namespace interop { namespace logic { namespace table } /** Zero out first column of every row * - * @param beg start of collection - * @param end end of collection - * @param column_count number of columns in of collection + * @param beg iterator to start of colleciton + * @param end iterator to end of collection + * @param column_count number of columns */ template void zero_first_column(I beg, I end, size_t column_count) @@ -140,15 +143,18 @@ namespace illumina { namespace interop { namespace logic { namespace table template void create_imaging_table_data(const model::metrics::run_metrics& metrics, const std::vector& columns, - const std::map& row_offset, + const row_offset_map_t& row_offset, I data_beg, I data_end) { typedef typename model::metrics::run_metrics::id_t id_t; + typedef model::metric_base::metric_set< model::metrics::tile_metric > tile_metric_set_t; + + if(columns.empty())return; const size_t column_count = columns.back().column_count(); const constants::tile_naming_method naming_method = metrics.run_info().flowcell().naming_method(); - const size_t q20_idx = metric::index_for_q_value(metrics.get_set(), 20); - const size_t q30_idx = metric::index_for_q_value(metrics.get_set(), 30); + const size_t q20_idx = metric::index_for_q_value(metrics.get(), 20); + const size_t q30_idx = metric::index_for_q_value(metrics.get(), 30); std::vector cmap(model::table::ImagingColumnCount, std::numeric_limits::max()); for(size_t i=0;i data_end) - INTEROP_THROW(model::index_out_of_bounds_exception, "Table is larger than buffer"); + INTEROP_THROW(model::index_out_of_bounds_exception, "Table is larger than buffer: " + << (column_count*row_offset.size()) << " > " << std::distance(data_beg, data_end) + << " column_count: " << column_count << " row_offset.size()=" << row_offset.size()); zero_first_column(data_beg, data_beg+column_count*row_offset.size(), column_count); - populate_imaging_table_data_by_cycle(metrics.get_set(), + populate_imaging_table_data_by_cycle(metrics.get(), q20_idx, q30_idx, naming_method, @@ -167,7 +175,7 @@ namespace illumina { namespace interop { namespace logic { namespace table row_offset, column_count, data_beg, data_end); - populate_imaging_table_data_by_cycle(metrics.get_set(), + populate_imaging_table_data_by_cycle(metrics.get(), q20_idx, q30_idx, naming_method, @@ -176,7 +184,7 @@ namespace illumina { namespace interop { namespace logic { namespace table row_offset, column_count, data_beg, data_end); - populate_imaging_table_data_by_cycle(metrics.get_set(), + populate_imaging_table_data_by_cycle(metrics.get(), q20_idx, q30_idx, naming_method, @@ -185,7 +193,7 @@ namespace illumina { namespace interop { namespace logic { namespace table row_offset, column_count, data_beg, data_end); - populate_imaging_table_data_by_cycle(metrics.get_set(), + populate_imaging_table_data_by_cycle(metrics.get(), q20_idx, q30_idx, naming_method, @@ -194,7 +202,7 @@ namespace illumina { namespace interop { namespace logic { namespace table row_offset, column_count, data_beg, data_end); - populate_imaging_table_data_by_cycle(metrics.get_set(), + populate_imaging_table_data_by_cycle(metrics.get(), q20_idx, q30_idx, naming_method, @@ -203,19 +211,22 @@ namespace illumina { namespace interop { namespace logic { namespace table row_offset, column_count, data_beg, data_end); - const typename model::metrics::run_metrics::tile_metric_set_t& tile_metrics = - metrics.get_set(); - for(typename std::map::const_iterator it = row_offset.begin();it != row_offset.end();++it) + const tile_metric_set_t& tile_metrics = metrics.get(); + for(typename row_offset_map_t::const_iterator it = row_offset.begin();it != row_offset.end();++it) { const id_t tid = model::metric_base::base_cycle_metric::tile_hash_from_id(it->first); if (!tile_metrics.has_metric(tid)) continue; - const size_t cycle = static_cast(model::metric_base::base_cycle_metric::cycle_from_id(it->first)); + const id_t cycle = model::metric_base::base_cycle_metric::cycle_from_id(it->first); const size_t row = it->second; - const summary::read_cycle& read = cycle_to_read[cycle-1]; + INTEROP_ASSERTMSG(cycle <= cycle_to_read.size(), + cycle << " <= " << cycle_to_read.size() + << " tile: " << model::metric_base::base_cycle_metric::tile_from_id(it->first)); + const summary::read_cycle& read = cycle_to_read[static_cast(cycle-1)]; table_populator::populate(tile_metrics.get_metric(tid), read.number, q20_idx, q30_idx, + 0, naming_method, cmap, data_beg+row*column_count, data_end); @@ -225,14 +236,14 @@ namespace illumina { namespace interop { namespace logic { namespace table * * @param metrics collection of all run metrics * @param columns vector of table columns - * @param row_offset map between metric id and row offset in table + * @param row_offset row offset map * @param data_beg iterator to start of table data * @param n number of cells in the data table */ template void populate_imaging_table_data_t(const model::metrics::run_metrics& metrics, const std::vector& columns, - const std::map& row_offset, + const row_offset_map_t& row_offset, I data_beg, const size_t n) { create_imaging_table_data(metrics, columns, row_offset, data_beg, data_beg+n); @@ -247,7 +258,7 @@ namespace illumina { namespace interop { namespace logic { namespace table */ void populate_imaging_table_data(const model::metrics::run_metrics& metrics, const std::vector& columns, - const std::map& row_offset, + const row_offset_map_t& row_offset, float* data_beg, const size_t n) { @@ -260,7 +271,7 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param row_offset ordering for the rows */ void count_table_rows(const model::metrics::run_metrics& metrics, - std::map& row_offset) + row_offset_map_t& row_offset) { typedef model::metrics::run_metrics::cycle_metric_map_t cycle_metric_map_t; cycle_metric_map_t hash_set; @@ -285,16 +296,16 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param metrics source run metrics * @param table destination imaging table */ - void create_imaging_table(const model::metrics::run_metrics& metrics, model::table::imaging_table& table) + void create_imaging_table(model::metrics::run_metrics& metrics, model::table::imaging_table& table) throw(model::invalid_column_type, model::index_out_of_bounds_exception) { typedef model::table::imaging_table::column_vector_t column_vector_t; typedef model::table::imaging_table::data_vector_t data_vector_t; - typedef std::map row_offset_t; - row_offset_t row_offset; + row_offset_map_t row_offset; column_vector_t columns; create_imaging_table_columns(metrics, columns); + if(columns.empty())return; count_table_rows(metrics, row_offset); data_vector_t data(row_offset.size()*count_table_columns(columns), std::numeric_limits::quiet_NaN()); create_imaging_table_data(metrics, columns, row_offset, data.begin(), data.end()); diff --git a/src/interop/logic/table/create_imaging_table_columns.cpp b/src/interop/logic/table/create_imaging_table_columns.cpp index 119bb4777..0aa697399 100644 --- a/src/interop/logic/table/create_imaging_table_columns.cpp +++ b/src/interop/logic/table/create_imaging_table_columns.cpp @@ -21,23 +21,23 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param tile_hash map between the tile has and base metric * @param filled destination array that indicates whether a column should be filled */ - void determine_filled_columns(const model::metrics::run_metrics& metrics, + void determine_filled_columns(model::metrics::run_metrics& metrics, const model::metrics::run_metrics::tile_metric_map_t& tile_hash, std::vector< bool >& filled) { typedef model::metrics::run_metrics::tile_metric_map_t tile_metric_map_t; filled.assign(model::table::ImagingColumnCount, false); - check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); - check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); - check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); - check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); - check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get(), filled); const model::metric_base::metric_set& tile_metrics = - metrics.get_set(); + metrics.get(); const constants::tile_naming_method naming_method = metrics.run_info().flowcell().naming_method(); - const size_t q20_idx = metric::index_for_q_value(metrics.get_set(), 20); - const size_t q30_idx = metric::index_for_q_value(metrics.get_set(), 30); + const size_t q20_idx = metric::index_for_q_value(metrics.get(), 20); + const size_t q30_idx = metric::index_for_q_value(metrics.get(), 30); for(tile_metric_map_t::const_iterator it = tile_hash.begin();it != tile_hash.end();++it) { if(!tile_metrics.has_metric(it->first)) continue; @@ -47,6 +47,7 @@ namespace illumina { namespace interop { namespace logic { namespace table read_it->number(), q20_idx, q30_idx, + 0, naming_method, filled); } @@ -95,7 +96,8 @@ namespace illumina { namespace interop { namespace logic { namespace table const std::vector& filled, std::vector< model::table::imaging_column >& columns) throw(model::invalid_column_type, - model::index_out_of_bounds_exception) + model::index_out_of_bounds_exception, + model::invalid_channel_exception) { columns.clear(); columns.reserve(model::table::ImagingColumnCount+channels.size()+constants::NUM_OF_BASES); @@ -157,16 +159,30 @@ namespace illumina { namespace interop { namespace logic { namespace table * @param metrics source collection of InterOp metrics from the run * @param columns destination vector of column descriptors */ - void create_imaging_table_columns(const model::metrics::run_metrics& metrics, - std::vector< model::table::imaging_column >& columns) - throw(model::invalid_column_type, model::index_out_of_bounds_exception) + void create_imaging_table_columns(model::metrics::run_metrics& metrics, + std::vector< model::table::imaging_column >& columns) + throw(model::invalid_column_type, + model::index_out_of_bounds_exception, + model::invalid_channel_exception) { typedef model::metrics::run_metrics::tile_metric_map_t tile_metric_map_t; std::vector< bool > filled; tile_metric_map_t tile_hash; metrics.populate_id_map(tile_hash); determine_filled_columns(metrics, tile_hash, filled); - create_imaging_table_columns(metrics.run_info().channels(), filled, columns); + create_imaging_table_columns(metrics.run_info().channels(), + filled, + columns); + } + + + /** Get the maximum number of digits to round + * + * @return maximum number of rounding digits + */ + ::uint32_t max_digits() + { + return check_imaging_table_column::max_digits(); } diff --git a/src/interop/model/metrics/corrected_intensity_metric.cpp b/src/interop/model/metrics/corrected_intensity_metric.cpp index 191e7a20d..bf0903e86 100644 --- a/src/interop/model/metrics/corrected_intensity_metric.cpp +++ b/src/interop/model/metrics/corrected_intensity_metric.cpp @@ -16,206 +16,208 @@ using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ namespace io { +namespace illumina{ namespace interop{ namespace io +{ #pragma pack(1) - /** Corrected Intensity Metric Record Layout Version 2 - * - * This class provides an interface to reading the corrected intensity metric file: - * - InterOp/CorrectedIntMetrics.bin - * - InterOp/CorrectedIntMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: corrected_intensity_metric - * 2. Version: 2 - */ - template<> - struct generic_layout : public default_layout<2> - { - /** @page corrected_v2 Corrected Intensity Version 2 - * - * - * This class provides an interface to reading the corrected intensity metric file: - * - InterOp/CorrectedIntMetrics.bin - * - InterOp/CorrectedIntMetricsOut.bin - * - * The file format for corrected intensity metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 2 bytes: average cycle intensity (uint16) - * 2 bytes: average corrected intensity for channel A (uint16) - * 2 bytes: average corrected intensity for channel C (uint16) - * 2 bytes: average corrected intensity for channel G (uint16) - * 2 bytes: average corrected intensity for channel T (uint16) - * 2 bytes: average corrected int for called clusters for base A (uint16) - * 2 bytes: average corrected int for called clusters for base C (uint16) - * 2 bytes: average corrected int for called clusters for base G (uint16) - * 2 bytes: average corrected int for called clusters for base T (uint16) - * 4 bytes: number of base calls for No Call (uint32) - * 4 bytes: number of base calls for base A (uint32) - * 4 bytes: number of base calls for base C (uint32) - * 4 bytes: number of base calls for base G (uint32) - * 4 bytes: number of base calls for base T (uint32) - * 4 bytes: signal to noise ratio (float32) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Intensity type */ - typedef ::uint16_t intensity_t; - /** Count type */ - typedef ::uint32_t count_t; - /** Map reading/writing to stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) - { - std::streamsize count = 0; - count += stream_map< intensity_t >(stream, metric.m_average_cycle_intensity); - count += stream_map< intensity_t >(stream, metric.m_corrected_int_all, constants::NUM_OF_BASES); - count += stream_map< intensity_t >(stream, metric.m_corrected_int_called, constants::NUM_OF_BASES); - count += stream_map< count_t >(stream, metric.m_called_counts, constants::NUM_OF_BASES_AND_NC); - count += stream_map< float >(stream, metric.m_signal_to_noise); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const corrected_intensity_metric::header_type&) - { - return static_cast( - sizeof(metric_id_t)+ - sizeof(intensity_t) + // m_average_cycle_intensity - sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_all - sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_called - sizeof(count_t)*constants::NUM_OF_BASES_AND_NC + // m_called_counts - sizeof(float) // m_signal_to_noise - ); - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const corrected_intensity_metric::header_type&) - { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); - } - }; - /** Corrected Intensity Metric Record Layout Version 3 - * - * This class provides an interface to reading the corrected intensity metric file: - * - InterOp/CorrectedIntMetrics.bin - * - InterOp/CorrectedIntMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: corrected_intensity_metric - * 2. Version: 3 - */ - template<> - struct generic_layout : public default_layout<3> - { - /** @page corrected_v3 Corrected Intensity Version 3 - * - * This class provides an interface to reading the corrected intensity metric file: - * - InterOp/CorrectedIntMetrics.bin - * - InterOp/CorrectedIntMetricsOut.bin - * - * The file format for corrected intensity metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 2 bytes: average corrected int for called clusters for base A (uint16) - * 2 bytes: average corrected int for called clusters for base C (uint16) - * 2 bytes: average corrected int for called clusters for base G (uint16) - * 2 bytes: average corrected int for called clusters for base T (uint16) - * 4 bytes: number of base calls for No Call (uint32) - * 4 bytes: number of base calls for base A (uint32) - * 4 bytes: number of base calls for base C (uint32) - * 4 bytes: number of base calls for base G (uint32) - * 4 bytes: number of base calls for base T (uint32) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Intensity type */ - typedef ::uint16_t intensity_t; - /** Count type */ - typedef ::uint32_t count_t; - /** Map reading/writing to stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) - { - std::streamsize count = 0; - // TODO: this does not need header layout, audit rest to see if that is true - count += stream_map< intensity_t >(stream, metric.m_corrected_int_called, constants::NUM_OF_BASES); - count += stream_map< count_t >(stream, metric.m_called_counts, constants::NUM_OF_BASES_AND_NC); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const corrected_intensity_metric::header_type&) - { - return static_cast( - sizeof(metric_id_t)+ - sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_called - sizeof(count_t)*constants::NUM_OF_BASES_AND_NC // m_called_counts - ); - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const corrected_intensity_metric::header_type&) - { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); - } - }; + /** Corrected Intensity Metric Record Layout Version 2 + * + * This class provides an interface to reading the corrected intensity metric file: + * - InterOp/CorrectedIntMetrics.bin + * - InterOp/CorrectedIntMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: corrected_intensity_metric + * 2. Version: 2 + */ + template<> + struct generic_layout : public default_layout<2> + { + /** @page corrected_v2 Corrected Intensity Version 2 + * + * + * This class provides an interface to reading the corrected intensity metric file: + * - InterOp/CorrectedIntMetrics.bin + * - InterOp/CorrectedIntMetricsOut.bin + * + * The file format for corrected intensity metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (2) + * byte 1: record size (48) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 2 bytes: average cycle intensity (uint16) + * 2 bytes: average corrected intensity for channel A (uint16) + * 2 bytes: average corrected intensity for channel C (uint16) + * 2 bytes: average corrected intensity for channel G (uint16) + * 2 bytes: average corrected intensity for channel T (uint16) + * 2 bytes: average corrected int for called clusters for base A (uint16) + * 2 bytes: average corrected int for called clusters for base C (uint16) + * 2 bytes: average corrected int for called clusters for base G (uint16) + * 2 bytes: average corrected int for called clusters for base T (uint16) + * 4 bytes: number of base calls for No Call (uint32) + * 4 bytes: number of base calls for base A (uint32) + * 4 bytes: number of base calls for base C (uint32) + * 4 bytes: number of base calls for base G (uint32) + * 4 bytes: number of base calls for base T (uint32) + * 4 bytes: signal to noise ratio (float32) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Intensity type */ + typedef ::uint16_t intensity_t; + /** Count type */ + typedef ::uint32_t count_t; + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) + { + std::streamsize count = 0; + count += stream_map< intensity_t >(stream, metric.m_average_cycle_intensity); + count += stream_map< intensity_t >(stream, metric.m_corrected_int_all, constants::NUM_OF_BASES); + count += stream_map< intensity_t >(stream, metric.m_corrected_int_called, constants::NUM_OF_BASES); + count += stream_map< count_t >(stream, metric.m_called_counts, constants::NUM_OF_BASES_AND_NC); + count += stream_map< float >(stream, metric.m_signal_to_noise); + return count; + } + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const corrected_intensity_metric::header_type&) + { + return static_cast( + sizeof(metric_id_t)+ + sizeof(intensity_t) + // m_average_cycle_intensity + sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_all + sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_called + sizeof(count_t)*constants::NUM_OF_BASES_AND_NC + // m_called_counts + sizeof(float) // m_signal_to_noise + ); + } + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const corrected_intensity_metric::header_type&) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + }; + /** Corrected Intensity Metric Record Layout Version 3 + * + * This class provides an interface to reading the corrected intensity metric file: + * - InterOp/CorrectedIntMetrics.bin + * - InterOp/CorrectedIntMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: corrected_intensity_metric + * 2. Version: 3 + */ + template<> + struct generic_layout : public default_layout<3> + { + /** @page corrected_v3 Corrected Intensity Version 3 + * + * This class provides an interface to reading the corrected intensity metric file: + * - InterOp/CorrectedIntMetrics.bin + * - InterOp/CorrectedIntMetricsOut.bin + * + * The file format for corrected intensity metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (3) + * byte 1: record size (34) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 2 bytes: average corrected int for called clusters for base A (uint16) + * 2 bytes: average corrected int for called clusters for base C (uint16) + * 2 bytes: average corrected int for called clusters for base G (uint16) + * 2 bytes: average corrected int for called clusters for base T (uint16) + * 4 bytes: number of base calls for No Call (uint32) + * 4 bytes: number of base calls for base A (uint32) + * 4 bytes: number of base calls for base C (uint32) + * 4 bytes: number of base calls for base G (uint32) + * 4 bytes: number of base calls for base T (uint32) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Intensity type */ + typedef ::uint16_t intensity_t; + /** Count type */ + typedef ::uint32_t count_t; + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) + { + std::streamsize count = 0; + // TODO: this does not need header layout, audit rest to see if that is true + count += stream_map< intensity_t >(stream, metric.m_corrected_int_called, constants::NUM_OF_BASES); + count += stream_map< count_t >(stream, metric.m_called_counts, constants::NUM_OF_BASES_AND_NC); + return count; + } + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const corrected_intensity_metric::header_type&) + { + return static_cast( + sizeof(metric_id_t)+ + sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_called + sizeof(count_t)*constants::NUM_OF_BASES_AND_NC // m_called_counts + ); + } + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const corrected_intensity_metric::header_type&) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + }; + #pragma pack() }}} diff --git a/src/interop/model/metrics/error_metric.cpp b/src/interop/model/metrics/error_metric.cpp index b41effc5e..0a1a3250a 100644 --- a/src/interop/model/metrics/error_metric.cpp +++ b/src/interop/model/metrics/error_metric.cpp @@ -14,98 +14,99 @@ using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ namespace io { +namespace illumina{ namespace interop{ namespace io +{ #pragma pack(1) - /** Error Metric Record Layout Version 3 - * - * This class provides an interface to reading the error metric file: - * - InterOp/ErrorMetrics.bin - * - InterOp/ErrorMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: error_metric - * 2. Version: 3 - */ - template<> - struct generic_layout : public default_layout<3> - { - /** @page error_v3 Error Version 3 - * - * This class provides an interface to reading the error metric file: - * - InterOp/ErrorMetrics.bin - * - InterOp/ErrorMetricsOut.bin - * - * The file format for error metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 4 bytes: error rate (float32) - * 4 bytes: number of perfect reads (uint32) - * 4 bytes: number of reads with 1 error (uint32) - * 4 bytes: number of reads with 2 error (uint32) - * 4 bytes: number of reads with 3 error (uint32) - * 4 bytes: number of reads with 4 error (uint32) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Error type */ - typedef float error_t; - /** Error type */ - typedef ::uint32_t count_t; - /** Map reading/writing to stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) - { - std::streamsize count = 0; - count += stream_map< error_t >(stream, metric.m_error_rate); - count += stream_map< count_t >(stream, metric.m_mismatch_cluster_count, error_metric::MAX_MISMATCH); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const error_metric::header_type&) - { - return static_cast(sizeof(metric_id_t)+ - sizeof(error_t)+ // m_error_rate - sizeof(count_t)*error_metric::MAX_MISMATCH // m_mismatch_cluster_count - ); - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const error_metric::header_type&) - { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); - } + /** Error Metric Record Layout Version 3 + * + * This class provides an interface to reading the error metric file: + * - InterOp/ErrorMetrics.bin + * - InterOp/ErrorMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: error_metric + * 2. Version: 3 + */ + template<> + struct generic_layout : public default_layout<3> + { + /** @page error_v3 Error Version 3 + * + * This class provides an interface to reading the error metric file: + * - InterOp/ErrorMetrics.bin + * - InterOp/ErrorMetricsOut.bin + * + * The file format for error metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (3) + * byte 1: record size (30) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 4 bytes: error rate (float32) + * 4 bytes: number of perfect reads (uint32) + * 4 bytes: number of reads with 1 error (uint32) + * 4 bytes: number of reads with 2 error (uint32) + * 4 bytes: number of reads with 3 error (uint32) + * 4 bytes: number of reads with 4 error (uint32) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Error type */ + typedef float error_t; + /** Error type */ + typedef ::uint32_t count_t; + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) + { + std::streamsize count = 0; + count += stream_map< error_t >(stream, metric.m_error_rate); + count += stream_map< count_t >(stream, metric.m_mismatch_cluster_count, error_metric::MAX_MISMATCH); + return count; + } + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const error_metric::header_type&) + { + return static_cast(sizeof(metric_id_t)+ + sizeof(error_t)+ // m_error_rate + sizeof(count_t)*error_metric::MAX_MISMATCH // m_mismatch_cluster_count + ); + } + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const error_metric::header_type&) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } - }; + }; #pragma pack() diff --git a/src/interop/model/metrics/extraction_metric.cpp b/src/interop/model/metrics/extraction_metric.cpp index bd557e036..7fa8c587f 100644 --- a/src/interop/model/metrics/extraction_metric.cpp +++ b/src/interop/model/metrics/extraction_metric.cpp @@ -19,138 +19,148 @@ using namespace illumina::interop::model::metrics; namespace illumina { namespace interop { namespace io { #pragma pack(1) - /** Extraction Metric Record Layout Version 2 - * - * This class provides an interface to reading the extraction metric file: - * - InterOp/ExtractionMetrics.bin - * - InterOp/ExtractionMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: extraction_metric - * 2. Version: 2 - */ - template<> - struct generic_layout : public default_layout<2> - { - /** @page extraction_v2 Extraction Version 2 - * - * This class provides an interface to reading the extraction metric file: - * - InterOp/ExtractionMetrics.bin - * - InterOp/ExtractionMetricsOut.bin - * - * The file format for extraction metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 4 bytes: focus for channel A (float32) - * 4 bytes: focus for channel C (float32) - * 4 bytes: focus for channel G (float32) - * 4 bytes: focus for channel T (float32) - * 2 bytes: max intensity for channel A (uint16) - * 2 bytes: max intensity for channel C (uint16) - * 2 bytes: max intensity for channel G (uint16) - * 2 bytes: max intensity for channel T (uint16) - * 8 bytes: date time stamp (uint64) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Intensity type */ - typedef float focus_t; - /** Intensity type */ - typedef ::uint16_t intensity_t; - /** Time type */ - typedef ::uint64_t datetime_t; - /** Map reading/writing to stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) - { - std::streamsize count = 0; - count += stream_map< focus_t >(stream, metric.m_focus_scores, extraction_metric::MAX_CHANNELS); - if(stream) - set_nan_to_zero(stream, metric.m_focus_scores);// TODO: Remove and rebaseline regression tests - else return count; - count += stream_map< intensity_t >(stream, metric.m_max_intensity_values, extraction_metric::MAX_CHANNELS); - count += stream_map< datetime_t >(stream, metric.m_date_time_csharp.value); - convert_datetime(stream, metric); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const extraction_metric::header_type&) - { - return static_cast( - sizeof(metric_id_t)+ - sizeof(focus_t)*extraction_metric::MAX_CHANNELS + // m_focus_scores - sizeof(intensity_t)*extraction_metric::MAX_CHANNELS+ // m_max_intensity_values - sizeof(datetime_t) // m_dateTime - ); - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const extraction_metric::header_type&) - { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); - } - private: - static void convert_datetime(std::ostream&, const extraction_metric&) - { - } - static void convert_datetime(const char*, extraction_metric& metric) - { - metric.m_date_time = metric.m_date_time_csharp.to_unix(); - } - static void convert_datetime(std::istream& in, extraction_metric& metric) - { - if(in.fail()) return; - metric.m_date_time = metric.m_date_time_csharp.to_unix(); - } - static void set_nan_to_zero(std::ostream&, const std::vector&) // TODO: Remove and rebaseline - { - } - static void set_nan_to_zero(std::istream&, std::vector& vals) - { - for(size_t i=0;i& vals) - { - for(size_t i=0;i + struct generic_layout : public default_layout<2> + { + /** @page extraction_v2 Extraction Version 2 + * + * This class provides an interface to reading the extraction metric file: + * - InterOp/ExtractionMetrics.bin + * - InterOp/ExtractionMetricsOut.bin + * + * The file format for extraction metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (2) + * byte 1: record size (38) + * + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 4 bytes: focus for channel A (float32) + * 4 bytes: focus for channel C (float32) + * 4 bytes: focus for channel G (float32) + * 4 bytes: focus for channel T (float32) + * 2 bytes: max intensity for channel A (uint16) + * 2 bytes: max intensity for channel C (uint16) + * 2 bytes: max intensity for channel G (uint16) + * 2 bytes: max intensity for channel T (uint16) + * 8 bytes: date time stamp (uint64) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Intensity type */ + typedef float focus_t; + /** Intensity type */ + typedef ::uint16_t intensity_t; + /** Time type */ + typedef ::uint64_t datetime_t; + + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream &stream, Metric &metric, Header &, const bool) + { + std::streamsize count = 0; + count += stream_map(stream, metric.m_focus_scores, extraction_metric::MAX_CHANNELS); + if (stream) + set_nan_to_zero(stream, metric.m_focus_scores);// TODO: Remove and rebaseline regression tests + else return count; + count += stream_map(stream, metric.m_max_intensity_values, extraction_metric::MAX_CHANNELS); + count += stream_map(stream, metric.m_date_time_csharp.value); + convert_datetime(stream, metric); + return count; + } + + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const extraction_metric::header_type &) + { + return static_cast( + sizeof(metric_id_t) + + sizeof(focus_t) * extraction_metric::MAX_CHANNELS + // m_focus_scores + sizeof(intensity_t) * extraction_metric::MAX_CHANNELS + // m_max_intensity_values + sizeof(datetime_t) // m_dateTime + ); + } + + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const extraction_metric::header_type &) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + + private: + static void convert_datetime(std::ostream &, const extraction_metric &) + { + // We do not support writing this date/time in C# format from C++ + } + static void convert_datetime(const char*, extraction_metric& metric) + { + metric.m_date_time = metric.m_date_time_csharp.to_unix(); + } + + static void convert_datetime(std::istream &in, extraction_metric &metric) + { + if (in.fail()) return; + metric.m_date_time = metric.m_date_time_csharp.to_unix(); + } + static void set_nan_to_zero(const char*, std::vector& vals) + { + for(size_t i=0;i &) // TODO: Remove and rebaseline + { + + } + static void set_nan_to_zero(std::istream&, std::vector& vals) + { + for(size_t i=0;i - struct generic_layout : public default_layout<1, 1 /*Multi record */> - { - /** @page image_v1 Image Version 1 - * - * This class provides an interface to reading the image metric file: - * - InterOp/ImageMetrics.bin - * - InterOp/ImageMetricsOut.bin - * - * The file format for image metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 2 bytes: channel number (uint16) - * 2 bytes: minimum contrast (uint16) - * 2 bytes: maximum contrast (uint16) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Record type */ - typedef generic_layout record_t; - /** Channel type */ - typedef ::uint16_t channel_t; - /** Contrast type */ - typedef ::uint16_t contrast_t; - /** Channel */ - channel_t channel; - /** Minimum contrast */ - contrast_t min_contrast; - /** Maximum contrast */ - contrast_t max_contrast; - /** Read metric from the input stream - * - * @param stream input stream - * @param metric destination metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(std::istream& stream, Metric& metric, Header&, const bool) - { - record_t rec; - std::streamsize count = stream_map< record_t >(stream, rec); - if( stream.fail() ) return count; - metric.m_channel_count = image_metric::MAX_CHANNELS; - INTEROP_ASSERTMSG(rec.channel < image_metric::MAX_CHANNELS, metric.lane() << "_" << metric.tile()<<" - " << rec.channel); - metric.m_min_contrast[rec.channel] = rec.min_contrast; - metric.m_max_contrast[rec.channel] = rec.max_contrast; - return count; - } - template - static std::streamsize map_stream(const char*, Metric&, Header&, const bool) - { - INTEROP_THROW(std::runtime_error, "This function should not be called"); - } - /** Write metric to the output stream - * - * @param stream output stream - * @param metric source metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(std::ostream& stream, Metric& metric, Header&, const bool) - { - std::streamsize count = 0; - metric_id_t metric_id; - metric_id.set(metric); - for(image_metric::ushort_t channel_index=0;channel_index < image_metric::MAX_CHANNELS;++channel_index) - { - if(channel_index > 0) write_binary(stream, metric_id); - count += stream_map< channel_t >(stream, channel_index); - count += stream_map< contrast_t >(stream, metric.m_min_contrast[channel_index]); - count += stream_map< contrast_t >(stream, metric.m_max_contrast[channel_index]); - } - return count; - } - /** Compute the size of a single metric record - * - * @return record size - */ - static record_size_t compute_size(const image_metric::header_type&) - { - return static_cast< record_size_t >( - sizeof(metric_id_t)+sizeof(record_t) - ); - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const image_metric::header_type&) - { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); - } - }; - /** Image Metric Record Layout Version 2 - * - * This class provides an interface to reading the image metric file: - * - InterOp/ImageMetrics.bin - * - InterOp/ImageMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: image_metric - * 2. Version: 2 - */ - template<> - struct generic_layout : public default_layout<2> + /** Image Metric Record Layout Version 1 + * + * This class provides an interface to reading the image metric file: + * - InterOp/ImageMetrics.bin + * - InterOp/ImageMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: image_metric + * 2. Version: 1 + */ + template<> + struct generic_layout : public default_layout<1, 1 /*Multi record */> + { + /** @page image_v1 Image Version 1 + * + * This class provides an interface to reading the image metric file: + * - InterOp/ImageMetrics.bin + * - InterOp/ImageMetricsOut.bin + * + * The file format for image metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (1) + * byte 1: record size (12) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 2 bytes: channel number (uint16) + * 2 bytes: minimum contrast (uint16) + * 2 bytes: maximum contrast (uint16) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Record type */ + typedef generic_layout record_t; + /** Channel type */ + typedef ::uint16_t channel_t; + /** Contrast type */ + typedef ::uint16_t contrast_t; + /** Channel */ + channel_t channel; + /** Minimum contrast */ + contrast_t min_contrast; + /** Maximum contrast */ + contrast_t max_contrast; + + /** Read metric from the input stream + * + * @param stream input stream + * @param metric destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(std::istream &stream, Metric &metric, Header &, const bool) + { + record_t rec; + std::streamsize count = stream_map(stream, rec); + if (stream.fail()) return count; + metric.m_channel_count = image_metric::MAX_CHANNELS; + INTEROP_ASSERTMSG(rec.channel < image_metric::MAX_CHANNELS, + metric.lane() << "_" << metric.tile() << " - " << rec.channel); + metric.m_min_contrast[rec.channel] = rec.min_contrast; + metric.m_max_contrast[rec.channel] = rec.max_contrast; + return count; + } + + /** Write metric to the output stream + * + * @param stream output stream + * @param metric source metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(std::ostream &stream, Metric &metric, Header &, const bool) + { + std::streamsize count = 0; + metric_id_t metric_id; + metric_id.set(metric); + for (image_metric::ushort_t channel_index = 0; channel_index < image_metric::MAX_CHANNELS; ++channel_index) { - /** @page image_v2 Image Version 2 - * - * This class provides an interface to reading the image metric file: - * - InterOp/ImageMetrics.bin - * - InterOp/ImageMetricsOut.bin - * - * The file format for image metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b Extended Header - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * byte 2: channel count (uint8) - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 2*channelCount bytes: minimum contrast (uint16) - * 2*channelCount bytes: maximum contrast (uint16) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Contrast type */ - typedef ::uint16_t contrast_t; - /** Contrast type */ - typedef ::uint8_t channel_count_t; + if (channel_index > 0) write_binary(stream, metric_id); + count += stream_map(stream, channel_index); + count += stream_map(stream, metric.m_min_contrast[channel_index]); + count += stream_map(stream, metric.m_max_contrast[channel_index]); + } + return count; + } + + /** Throws an unimplemented error + */ + template + static std::streamsize map_stream(const char *, const Metric &, const Header &, const bool) + { + INTEROP_THROW(std::runtime_error, "Function not implemented"); + } + + /** Compute the size of a single metric record + * + * @return record size + */ + static record_size_t compute_size(const image_metric::header_type &) + { + return static_cast< record_size_t >( + sizeof(metric_id_t) + sizeof(record_t) + ); + } + + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const image_metric::header_type &) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + }; + + /** Image Metric Record Layout Version 2 + * + * This class provides an interface to reading the image metric file: + * - InterOp/ImageMetrics.bin + * - InterOp/ImageMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: image_metric + * 2. Version: 2 + */ + template<> + struct generic_layout : public default_layout<2> + { + /** @page image_v2 Image Version 2 + * + * This class provides an interface to reading the image metric file: + * - InterOp/ImageMetrics.bin + * - InterOp/ImageMetricsOut.bin + * + * The file format for image metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (2) + * byte 1: record size (6 + 2*channelCount + 2*channelCount) + * + * @b Extended Header + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * byte 2: channel count (uint8) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 2*channelCount bytes: minimum contrast (uint16) + * 2*channelCount bytes: maximum contrast (uint16) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Contrast type */ + typedef ::uint16_t contrast_t; + /** Contrast type */ + typedef ::uint8_t channel_count_t; + + /** Map reading/writing a metric to a stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @param header metric header layout + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream &stream, Metric &metric, Header &header, const bool) + { + std::streamsize count = 0; + if (header.m_channel_count == 0) + INTEROP_THROW(bad_format_exception, "Cannot write data where channel count is 0"); + copy_from(stream, metric.m_channel_count, header.m_channel_count); + count += stream_map(stream, metric.m_min_contrast, header.m_channel_count); + count += stream_map(stream, metric.m_max_contrast, header.m_channel_count); + return count; + } + + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const image_metric::header_type &header) + { + return static_cast(sizeof(metric_id_t) + header.channel_count() * sizeof(contrast_t) * 2); + } + + /** Map reading/writing a header to a stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param header source/destination header + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream_for_header(Stream &stream, Header &header) + { + std::streamsize count = stream_map(stream, header.m_channel_count); + if (stream.fail())return count; + if (header.m_channel_count == 0) + INTEROP_THROW(bad_format_exception, "Cannot write data where channel count is 0"); + return count; + } + + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const image_metric::header_type &) + { + return static_cast(sizeof(channel_count_t) + sizeof(record_size_t) + sizeof(version_t)); + } + }; + - /** Map reading/writing a metric to a stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @param header metric header layout - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header& header, const bool) - { - std::streamsize count = 0; - if(header.m_channel_count==0) - INTEROP_THROW(bad_format_exception, "Cannot write data where channel count is 0"); - copy_from(stream, metric.m_channel_count, header.m_channel_count); - count += stream_map< contrast_t >(stream, metric.m_min_contrast, header.m_channel_count); - count += stream_map< contrast_t >(stream, metric.m_max_contrast, header.m_channel_count); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const image_metric::header_type& header) - { - return static_cast(sizeof(metric_id_t)+header.channel_count()*sizeof(contrast_t)*2); - } - /** Map reading/writing a header to a stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param header source/destination header - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream_for_header(Stream& stream, Header& header) - { - std::streamsize count = stream_map< channel_count_t >(stream, header.m_channel_count); - if(stream.fail()) return count; - if(header.m_channel_count==0) - INTEROP_THROW(bad_format_exception, "Cannot write data where channel count is 0"); - return count; - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const image_metric::header_type&) - { - return static_cast(sizeof(channel_count_t) + sizeof(record_size_t) + sizeof(version_t)); - } - }; #pragma pack() // DO NOT MOVE - }}} +}}} INTEROP_FORCE_LINK_DEF(image_metric) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 1 ) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 2 ) + +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 1) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 2) diff --git a/src/interop/model/metrics/index_metric.cpp b/src/interop/model/metrics/index_metric.cpp index 20c2c89a7..659fc1441 100644 --- a/src/interop/model/metrics/index_metric.cpp +++ b/src/interop/model/metrics/index_metric.cpp @@ -59,18 +59,19 @@ namespace illumina { namespace interop { namespace io * * 2 bytes: index name length (indexNameLength) (uint16) * indexNameLength bytes: index name - * 4 bytes: index count (uint32) + * 4 bytes: index cluster count (uint32) * 2 bytes: sample name length (sampleNameLength) (uint16) * sampleNameLength bytes: sample name * 2 bytes: project name length (projectNameLength) (uint16) * projectNameLength bytes: project name */ /** Metric ID type */ - typedef layout::base_read_metric metric_id_t; + typedef layout::base_read_metric< ::uint16_t > metric_id_t; /** Defines an empty header */ typedef void *header_t; /** No record size is required for this stream */ //typedef fixed_record_size record_size_t; + typedef ::uint32_t cluster_count_t; enum { BYTE_COUNT = 1, RECORD_SIZE = sizeof(metric_id_t) + BYTE_COUNT @@ -86,27 +87,27 @@ namespace illumina { namespace interop { namespace io static std::streamsize map_stream(std::istream &in, Metric &metric, Header &, const bool) { std::string index_name; - ::uint32_t count; + cluster_count_t count; std::string sample_name; std::string project_name; - read_binary(in, index_name); + read_binary(in, index_name, "NA"); if (in.fail()) INTEROP_THROW(incomplete_file_exception, "No more data after index name"); read_binary(in, count); if (in.fail()) INTEROP_THROW(incomplete_file_exception, "No more data after count"); - read_binary(in, sample_name); + read_binary(in, sample_name, "NA"); if (in.fail()) INTEROP_THROW(incomplete_file_exception, "No more data after sample name"); - read_binary(in, project_name); + read_binary(in, project_name, "NA"); index_metric::index_array_t::iterator beg = metric.m_indices.begin(), end = metric.m_indices.end(); for (; beg != end; ++beg) if (beg->index_seq() == sample_name) break; if (beg == end) { metric.m_indices.push_back(index_info(index_name, sample_name, project_name, count)); } - else beg->m_count += count; + else beg->m_cluster_count += count; return BYTE_COUNT; } @@ -132,7 +133,7 @@ namespace illumina { namespace interop { namespace io { if (beg != metric.indices().begin()) write_binary(out, metric_id); write_binary(out, beg->index_seq()); - write_binary(out, static_cast< ::uint32_t >(beg->count())); + write_binary(out, static_cast< cluster_count_t >(beg->cluster_count())); write_binary(out, beg->sample_id()); write_binary(out, beg->sample_proj()); } diff --git a/src/interop/model/metrics/q_collapsed_metric.cpp b/src/interop/model/metrics/q_collapsed_metric.cpp index 713736598..07e87f80a 100644 --- a/src/interop/model/metrics/q_collapsed_metric.cpp +++ b/src/interop/model/metrics/q_collapsed_metric.cpp @@ -32,7 +32,7 @@ namespace illumina{ namespace interop{ namespace io { * 2. Version: 2 */ template<> - struct generic_layout : public default_layout<2, 1 /*Tmp hack */> + struct generic_layout : public default_layout<2> { /** @page q_collapsed_v2 Collapsed Q-Metrics Version 2 * @@ -46,7 +46,7 @@ namespace illumina{ namespace interop{ namespace io { * * illumina::interop::io::read_metrics (Function that parses this information) * - * byte 0: version number + * byte 0: version number (2) * byte 1: record size (can take two different values) * * @b n-Records @@ -65,7 +65,7 @@ namespace illumina{ namespace interop{ namespace io { * 4 bytes: Median score (uint32) */ /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; /** Histogram count type */ typedef ::uint32_t count_t; /** Histogram count type */ @@ -225,7 +225,7 @@ namespace illumina{ namespace interop{ namespace io { * * illumina::interop::io::read_metrics (Function that parses this information) * - * byte 0: version number + * byte 0: version number (3) * byte 1: record size * * @b n-Records @@ -273,7 +273,7 @@ namespace illumina{ namespace interop{ namespace io { * * illumina::interop::io::read_metrics (Function that parses this information) * - * byte 0: version number + * byte 0: version number (4) * byte 1: record size * * @b n-Records @@ -321,7 +321,7 @@ namespace illumina{ namespace interop{ namespace io { * * illumina::interop::io::read_metrics (Function that parses this information) * - * byte 0: version number + * byte 0: version number (5) * byte 1: record size * * @b Extended Header @@ -353,7 +353,7 @@ namespace illumina{ namespace interop{ namespace io { * 4 bytes: Median score (uint32) */ /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; /** Histogram count type */ typedef ::uint32_t count_t; /** Histogram count type */ @@ -448,6 +448,7 @@ namespace illumina{ namespace interop{ namespace io { INTEROP_ASSERTMSG(bin_count>0, VERSION); bin_t tmp[q_metric::MAX_Q_BINS]; count+=stream_map< bin_t >(stream, tmp, bin_count*3); + map_resize(header.m_qscore_bins, bin_count); return count; } @@ -455,9 +456,15 @@ namespace illumina{ namespace interop{ namespace io { * * @return header size */ - static record_size_t compute_header_size(const q_collapsed_metric::header_type&) + static record_size_t compute_header_size(const q_collapsed_metric::header_type& header) { - return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); + if (header.bin_count() == 0) + return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); + return static_cast(sizeof(record_size_t) + + sizeof(version_t) + // version + sizeof(bool_t) + // has bins + sizeof(bin_count_t) + // number of bins + header.bin_count() * 3 * sizeof(bin_t)); } /** Does not read/write record size, this is done in `map_stream_for_header` * @@ -527,8 +534,8 @@ namespace illumina{ namespace interop{ namespace io { * * illumina::interop::io::read_metrics (Function that parses this information) * - * byte 0: version number - * byte 1: record size + * byte 0: version number (6) + * byte 1: record size (22) * * @b Extended Header * diff --git a/src/interop/model/metrics/q_metric.cpp b/src/interop/model/metrics/q_metric.cpp index db6ef5644..ad4e928e7 100644 --- a/src/interop/model/metrics/q_metric.cpp +++ b/src/interop/model/metrics/q_metric.cpp @@ -23,547 +23,733 @@ namespace illumina { namespace interop { namespace io { #pragma pack(1) - /** Q-score Metric Record Layout Version 4 - * - * This class provides an interface to reading the q-metric file: - * - InterOp/QMetrics.bin - * - InterOp/QMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: q_metric - * 2. Version: 4 - */ - template<> - struct generic_layout : public default_layout<4> - { - /** @page q_v4 Q-Metrics Version 4 - * - * This class provides an interface to reading the q-metric file: - * - InterOp/QMetrics.bin - * - InterOp/QMetricsOut.bin - * - * The file format for q-metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 200 bytes: q-score histogram (uint32_t*50) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Histogram count type */ - typedef ::uint32_t count_t; - /** Map reading/writing to stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) - { - const std::streamsize count = stream_map< count_t >(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS); - resize_accumulated(stream, metric); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const q_metric::header_type&) - { - return static_cast(sizeof(metric_id_t)+sizeof(count_t)*q_metric::MAX_Q_BINS); - } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const q_metric::header_type&) - { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); - } - private: - static void resize_accumulated(const char*, q_metric& metric) - { - metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); - } - static void resize_accumulated(std::istream&, q_metric& metric) - { - metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); - } - static void resize_accumulated(std::ostream&, const q_metric&){} - }; - /** Q-score Metric Record Layout Version 5 - * - * This class provides an interface to reading the q-metric file: - * - InterOp/QMetrics.bin - * - InterOp/QMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: q_metric - * 2. Version: 5 - */ - template<> - struct generic_layout : public default_layout<5> + /** Q-score Metric Record Layout Version 4 + * + * This class provides an interface to reading the q-metric file: + * - InterOp/QMetrics.bin + * - InterOp/QMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: q_metric + * 2. Version: 4 + */ + template<> + struct generic_layout : public default_layout<4> + { + /** @page q_v4 Q-Metrics Version 4 + * + * This class provides an interface to reading the q-metric file: + * - InterOp/QMetrics.bin + * - InterOp/QMetricsOut.bin + * + * The file format for q-metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (4) + * byte 1: record size (206) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 200 bytes: q-score histogram (uint32_t*50) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Histogram count type */ + typedef ::uint32_t count_t; + + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream &stream, Metric &metric, Header &, const bool) + { + const std::streamsize count = stream_map(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS); + resize_accumulated(stream, metric); + return count; + } + + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const q_metric::header_type &) + { + return static_cast(sizeof(metric_id_t) + sizeof(count_t) * q_metric::MAX_Q_BINS); + } + + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const q_metric::header_type &) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + + private: + static void resize_accumulated(const char*, q_metric& metric) + { + metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); + } + static void resize_accumulated(std::istream &, q_metric &metric) + { + metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); + } + + static void resize_accumulated(std::ostream &, const q_metric &) + { } + }; + + /** Q-score Metric Record Layout Version 5 + * + * This class provides an interface to reading the q-metric file: + * - InterOp/QMetrics.bin + * - InterOp/QMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: q_metric + * 2. Version: 5 + */ + template<> + struct generic_layout : public default_layout<5> + { + /** @page q_v5 Q-Metrics Version 5 + * + * This class provides an interface to reading the q-metric file: + * - InterOp/QMetrics.bin + * - InterOp/QMetricsOut.bin + * + * The file format for q-metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (5) + * byte 1: record size (206) + * + * @b Extended Header + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * byte 2: flag indicating whether is has bins (bool) + * + * If byte 2 is true, then the following information is also in the header: + * + * byte 3: number of bins (uint8) + * byte 4-binCount: array of low ends for each bin (uint8) + * byte 4+binCount-4+binCount*2: array of high ends for each bin (uint8) + * byte 4+2*binCount-4+binCount*3: array of values for each (uint8) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 200 bytes: q-score histogram (uint32_t*50) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Histogram count type */ + typedef ::uint32_t count_t; + /** Bool type */ + typedef ::uint8_t bool_t; + /** Bin count type */ + typedef ::uint8_t bin_count_t; + /** Bin type */ + typedef ::uint8_t bin_t; + + /** Read metric from the input stream + * + * @param stream input stream + * @param metric destination metric + * @param header metric header + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream &stream, Metric &metric, Header &header, const bool) + { + if (header.m_qscore_bins.size() > 0) { - /** @page q_v5 Q-Metrics Version 5 - * - * This class provides an interface to reading the q-metric file: - * - InterOp/QMetrics.bin - * - InterOp/QMetricsOut.bin - * - * The file format for q-metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b Extended Header - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * byte 2: flag indicating whether is has bins (bool) - * - * If byte 2 is true, then the following information is also in the header: - * - * byte 3: number of bins (uint8) - * byte 4-binCount: array of low ends for each bin (uint8) - * byte 4+binCount-4+binCount*2: array of high ends for each bin (uint8) - * byte 4+2*binCount-4+binCount*3: array of values for each (uint8) - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 200 bytes: q-score histogram (uint32_t*50) - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Histogram count type */ - typedef ::uint32_t count_t; - /** Bool type */ - typedef ::uint8_t bool_t; - /** Bin count type */ - typedef ::uint8_t bin_count_t; - /** Bin type */ - typedef ::uint8_t bin_t; - - /** Read metric from the input stream - * - * @param stream input stream - * @param metric destination metric - * @param header metric header - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header& header, const bool) - { - if(header.m_qscore_bins.size()>0) - { - count_t hist[q_metric::MAX_Q_BINS]; - std::streamsize count = stream_map< count_t >(stream, hist, q_metric::MAX_Q_BINS); - map_resize(metric.m_qscore_hist, header.m_qscore_bins.size()); - for(size_t i=0;i 0), header.m_qscore_bins[i].value()); - INTEROP_ASSERTMSG((header.m_qscore_bins[i].value() - 1) < q_metric::MAX_Q_BINS, header.m_qscore_bins[i].value() - 1 << " < " << q_metric::MAX_Q_BINS); - INTEROP_ASSERTMSG(i< metric.m_qscore_hist.size(), metric.m_qscore_hist.size()); - metric.m_qscore_hist[i] = hist[header.m_qscore_bins[i].value()-1]; - } - resize_accumulated(stream, metric); - return count; - } - const std::streamsize count = stream_map< count_t >(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS); - resize_accumulated(stream, metric); - return count; - } - /** Write metric to the output stream - * - * @param stream output stream - * @param metric source metric - * @param header metric header - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(std::ostream& stream, Metric& metric, Header& header, const bool) - { - if(header.m_qscore_bins.size()>0) - { - count_t hist[q_metric::MAX_Q_BINS]; - std::fill(hist, hist+q_metric::MAX_Q_BINS, 0); - for(size_t i=0;i 0); - INTEROP_ASSERTMSG((header.m_qscore_bins[i].value() - 1) < q_metric::MAX_Q_BINS, header.m_qscore_bins[i].value() - 1 << " < " << q_metric::MAX_Q_BINS); - hist[header.m_qscore_bins[i].value()-1] = metric.m_qscore_hist[i]; - } - return stream_map< count_t >(stream, hist, q_metric::MAX_Q_BINS); - } - return stream_map< count_t >(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS); - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const q_metric::header_type&) + count_t hist[q_metric::MAX_Q_BINS]; + std::streamsize count = stream_map(stream, hist, q_metric::MAX_Q_BINS); + map_resize(metric.m_qscore_hist, header.m_qscore_bins.size()); + for (size_t i = 0; i < header.m_qscore_bins.size(); ++i) { - return static_cast(sizeof(metric_id_t)+sizeof(count_t)*q_metric::MAX_Q_BINS); + INTEROP_ASSERTMSG((header.m_qscore_bins[i].value() > 0), header.m_qscore_bins[i].value()); + INTEROP_ASSERTMSG((header.m_qscore_bins[i].value() - 1) < q_metric::MAX_Q_BINS, + header.m_qscore_bins[i].value() - 1 << " < " << q_metric::MAX_Q_BINS); + INTEROP_ASSERTMSG(i < metric.m_qscore_hist.size(), metric.m_qscore_hist.size()); + metric.m_qscore_hist[i] = hist[header.m_qscore_bins[i].value() - 1]; } - /** Map reading/writing a header to a stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param header source/destination header - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream_for_header(Stream& stream, Header& header) - { - bool_t has_bins = header.bin_count()>0; - std::streamsize count = 0; - count += stream_map< bool_t >(stream, has_bins); - if(stream.fail()) return count; - if(!has_bins) return count; - bin_count_t bin_count = static_cast(header.bin_count()); - count+=stream_map< bin_count_t>(stream, bin_count); - if(stream.fail()) return count; - if(bin_count==0) - INTEROP_THROW(bad_format_exception, "Zero bins is not supported"); - INTEROP_ASSERT(bin_count>0); - bin_t tmp[q_metric::MAX_Q_BINS]; - map_resize(header.m_qscore_bins, bin_count); - - copy_lower_write(header.m_qscore_bins, tmp); - count+=stream_map< bin_t >(stream, tmp, bin_count); - copy_lower_read(header.m_qscore_bins, tmp); - - copy_upper_write(header.m_qscore_bins, tmp); - count+=stream_map< bin_t >(stream, tmp, bin_count); - copy_upper_read(header.m_qscore_bins, tmp); - - copy_value_write(header.m_qscore_bins, tmp); - count+=stream_map< bin_t >(stream, tmp, bin_count); - copy_value_read(header.m_qscore_bins, tmp); - - return count; - } - /** Compute header size - * - * @param header q-metric header - * @return header size - */ - static record_size_t compute_header_size(const q_metric::header_type& header) - { - if(header.bin_count()==0) return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); - return static_cast(sizeof(record_size_t) + - sizeof(version_t) + // version - sizeof(bool_t) + // has bins - sizeof(bin_count_t) + // number of bins - header.bin_count()*3*sizeof(bin_t)); - } - private: - template - static void copy_lower_read(std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i&, bin_t*) { } - template - static void copy_lower_write(const std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i(bins[i].m_lower); - } - static void copy_lower_write(std::vector&, bin_t*) { } - private: - template - static void copy_upper_read(std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i& , bin_t*) { } - template - static void copy_upper_write(const std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i(bins[i].m_upper); - } - static void copy_upper_write(std::vector&, bin_t*) { } - private: - template - static void copy_value_read(std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i&, bin_t*) { } - template - static void copy_value_write(const std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i(bins[i].m_value); - } - static void copy_value_write(std::vector&, bin_t*) { } + resize_accumulated(stream, metric); + return count; + } + const std::streamsize count = stream_map(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS); + resize_accumulated(stream, metric); + return count; + } - private: - static void resize_accumulated(const char*, q_metric& metric) - { - metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); - } - static void resize_accumulated(std::istream&, q_metric& metric) - { - metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); - } - static void resize_accumulated(std::ostream&, const q_metric&){} - }; - /** Q-score Metric Record Layout Version 6 - * - * This class provides an interface to reading the q-metric file: - * - InterOp/QMetrics.bin - * - InterOp/QMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: q_metric - * 2. Version: 6 - */ - template<> - struct generic_layout : public default_layout<6> + /** Write metric to the output stream + * + * @param stream output stream + * @param metric source metric + * @param header metric header + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(std::ostream &stream, Metric &metric, Header &header, const bool) + { + if (header.m_qscore_bins.size() > 0) { - /** @page q_v6 Q-Metrics Version 6 - * - * This class provides an interface to reading the q-metric file: - * - InterOp/QMetrics.bin - * - InterOp/QMetricsOut.bin - * - * The file format for q-metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b Extended Header - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * byte 2: flag indicating whether is has bins (bool) - * - * If byte 2 is true, then the following information is also in the header: - * - * byte 3: number of bins (uint8) - * byte 4-binCount: array of low ends for each bin (uint8) - * byte 4+binCount-4+binCount*2: array of high ends for each bin (uint8) - * byte 4+2*binCount-4+binCount*3: array of values for each bin (uint8) - * - * @b n-Records - * - * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: cycle number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 4*binCount bytes: q-score histogram (uint32_t*binCount) - * - * Note, if the header has no bins, then binCount is 50 for the records - */ - /** Metric ID type */ - typedef layout::base_cycle_metric metric_id_t; - /** Histogram count type */ - typedef ::uint32_t count_t; - /** Bool type */ - typedef ::uint8_t bool_t; - /** Bin count type */ - typedef ::uint8_t bin_count_t; - /** Bin type */ - typedef ::uint8_t bin_t; - - /** Map reading/writing to stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param metric source/destination metric - * @param header metric header layout - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(Stream& stream, Metric& metric, Header& header, const bool) - { - const size_t bin_count = (header.bin_count() == 0) ? static_cast(q_metric::MAX_Q_BINS) : header.bin_count(); - if(bin_count==0) - INTEROP_THROW(bad_format_exception, "Zero bins is not supported"); - const std::streamsize count = stream_map< count_t >(stream, metric.m_qscore_hist, bin_count); - resize_accumulated(stream, metric); - return count; - } - /** Compute the layout size - * - * @return size of the record - */ - static record_size_t compute_size(const q_metric::header_type& header) - { - const size_t bin_count = header.bin_count() == 0 ? static_cast(q_metric::MAX_Q_BINS) : header.bin_count(); - return static_cast(sizeof(metric_id_t)+sizeof(count_t)*bin_count); - } - /** Map reading/writing a header to a stream - * - * Reading and writing are symmetric operations, map it once - * - * @param stream input/output stream - * @param header source/destination header - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream_for_header(Stream& stream, Header& header) - { - bool_t has_bins = header.bin_count()>0; - std::streamsize count = 0; - count += stream_map< bool_t >(stream, has_bins); - if(stream.fail()) return count; - if(!has_bins) return count; - bin_count_t bin_count = static_cast(header.bin_count()); - count+=stream_map< bin_count_t >(stream, bin_count); - if(stream.fail()) return count; - if(bin_count==0) - INTEROP_THROW(bad_format_exception, "Zero bins is not supported"); - INTEROP_ASSERT(bin_count>0); - bin_t tmp[q_metric::MAX_Q_BINS]; - map_resize(header.m_qscore_bins, bin_count); - - copy_lower_write(header.m_qscore_bins, tmp); - count+=stream_map< bin_t >(stream, tmp, bin_count); - copy_lower_read(header.m_qscore_bins, tmp); - - copy_upper_write(header.m_qscore_bins, tmp); - count+=stream_map< bin_t >(stream, tmp, bin_count); - copy_upper_read(header.m_qscore_bins, tmp); - - copy_value_write(header.m_qscore_bins, tmp); - count+=stream_map< bin_t >(stream, tmp, bin_count); - copy_value_read(header.m_qscore_bins, tmp); - - return count; - } - /** Compute header size - * - * @param header q-metric header - * @return header size - */ - static record_size_t compute_header_size(const q_metric::header_type& header) - { - if(header.bin_count()==0) - return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); - return static_cast(sizeof(record_size_t) + - sizeof(version_t) + // version - sizeof(bool_t) + // has bins - sizeof(bin_count_t) + // number of bins - header.bin_count()*3*sizeof(bin_t)); - } - private: - template - static void copy_lower_read(std::vector& bins, bin_t (&tmp)[N]) + count_t hist[q_metric::MAX_Q_BINS]; + std::fill(hist, hist + q_metric::MAX_Q_BINS, 0); + for (size_t i = 0; i < header.m_qscore_bins.size(); i++) { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i 0); + INTEROP_ASSERTMSG((header.m_qscore_bins[i].value() - 1) < q_metric::MAX_Q_BINS, + header.m_qscore_bins[i].value() - 1 << " < " << q_metric::MAX_Q_BINS); + hist[header.m_qscore_bins[i].value() - 1] = metric.m_qscore_hist[i]; } - static void copy_lower_read(const std::vector&, bin_t*) { } - template - static void copy_lower_write(const std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i(bins[i].m_lower); - } - static void copy_lower_write(std::vector&, bin_t*) { } - private: - template - static void copy_upper_read(std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i&, bin_t*) { } - template - static void copy_upper_write(const std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i(bins[i].m_upper); - } - static void copy_upper_write(std::vector&, bin_t*) { } - private: - template - static void copy_value_read(std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i&, bin_t*) { } - template - static void copy_value_write(const std::vector& bins, bin_t (&tmp)[N]) - { - INTEROP_ASSERT(bins.size() <= N); - for(size_t i=0;i(bins[i].m_value); - } - static void copy_value_write(std::vector&, bin_t*) { } + return stream_map(stream, hist, q_metric::MAX_Q_BINS); + } + return stream_map(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS); + } + /** Throws an unimplemented error + */ + template + static std::streamsize map_stream(const char*, const Metric &, const Header &, const bool) + { + INTEROP_THROW(std::runtime_error, "Function not implemented"); + } - private: - static void resize_accumulated(const char*, q_metric& metric) - { - metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); - } - static void resize_accumulated(std::istream&, q_metric& metric) - { - metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); - } - static void resize_accumulated(std::ostream&, const q_metric&){} - }; + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const q_metric::header_type &) + { + return static_cast(sizeof(metric_id_t) + sizeof(count_t) * q_metric::MAX_Q_BINS); + } + + /** Map reading/writing a header to a stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param header source/destination header + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream_for_header(Stream &stream, Header &header) + { + bool_t has_bins = header.bin_count() > 0; + std::streamsize count = 0; + count += stream_map(stream, has_bins); + if (stream.fail()) return count; + if (!has_bins) return count; + bin_count_t bin_count = static_cast(header.bin_count()); + count += stream_map(stream, bin_count); + if (stream.fail()) return count; + if(bin_count==0) + INTEROP_THROW(bad_format_exception, "Zero bins is not supported"); + INTEROP_ASSERT(bin_count > 0); + bin_t tmp[q_metric::MAX_Q_BINS]; + map_resize(header.m_qscore_bins, bin_count); + + copy_lower_write(header.m_qscore_bins, tmp); + count += stream_map(stream, tmp, bin_count); + copy_lower_read(header.m_qscore_bins, tmp); + + copy_upper_write(header.m_qscore_bins, tmp); + count += stream_map(stream, tmp, bin_count); + copy_upper_read(header.m_qscore_bins, tmp); + + copy_value_write(header.m_qscore_bins, tmp); + count += stream_map(stream, tmp, bin_count); + copy_value_read(header.m_qscore_bins, tmp); + + return count; + } + + /** Compute header size + * + * @param header q-metric header + * @return header size + */ + static record_size_t compute_header_size(const q_metric::header_type &header) + { + if (header.bin_count() == 0) + return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); + return static_cast(sizeof(record_size_t) + + sizeof(version_t) + // version + sizeof(bool_t) + // has bins + sizeof(bin_count_t) + // number of bins + header.bin_count() * 3 * sizeof(bin_t)); + } + + private: + template + static void copy_lower_read(std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) bins[i].m_lower = tmp[i]; + } + + static void copy_lower_read(const std::vector &, bin_t *) + { } + + template + static void copy_lower_write(const std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) tmp[i] = static_cast< bin_t >(bins[i].m_lower); + } + + static void copy_lower_write(std::vector &, bin_t *) + { } + + private: + template + static void copy_upper_read(std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) bins[i].m_upper = tmp[i]; + } + + static void copy_upper_read(const std::vector &, bin_t *) + { } + + template + static void copy_upper_write(const std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) tmp[i] = static_cast< bin_t >(bins[i].m_upper); + } + + static void copy_upper_write(std::vector &, bin_t *) + { } + + private: + template + static void copy_value_read(std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) bins[i].m_value = tmp[i]; + } + + static void copy_value_read(const std::vector &, bin_t *) + { } + + template + static void copy_value_write(const std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) tmp[i] = static_cast< bin_t >(bins[i].m_value); + } + + static void copy_value_write(std::vector &, bin_t *) + { } + + private: + static void resize_accumulated(const char*, q_metric& metric) + { + metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); + } + static void resize_accumulated(std::istream &, q_metric &metric) + { + metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); + } + + static void resize_accumulated(std::ostream &, const q_metric &) + { } + }; + + /** Q-score Metric Record Layout Version 6 + * + * This class provides an interface to reading the q-metric file: + * - InterOp/QMetrics.bin + * - InterOp/QMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: q_metric + * 2. Version: 6 + */ + template<> + struct generic_layout : public default_layout<6> + { + /** @page q_v6 Q-Metrics Version 6 + * + * This class provides an interface to reading the q-metric file: + * - InterOp/QMetrics.bin + * - InterOp/QMetricsOut.bin + * + * The file format for q-metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (6) + * byte 1: record size (6 + 4*binCount if flag is true otherwise 206) + * + * @b Extended Header + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * byte 2: flag indicating whether is has bins (bool) + * + * If byte 2 is true, then the following information is also in the header: + * + * byte 3: number of bins (uint8) + * byte 4-binCount: array of low ends for each bin (uint8) + * byte 4+binCount-4+binCount*2: array of high ends for each bin (uint8) + * byte 4+2*binCount-4+binCount*3: array of values for each bin (uint8) + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 4*binCount bytes: q-score histogram (uint32_t*binCount) + * + * Note, if the header has no bins, then binCount is 50 for the records + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint16_t > metric_id_t; + /** Histogram count type */ + typedef ::uint32_t count_t; + /** Bool type */ + typedef ::uint8_t bool_t; + /** Bin count type */ + typedef ::uint8_t bin_count_t; + /** Bin type */ + typedef ::uint8_t bin_t; + + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @param header metric header layout + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream &stream, Metric &metric, Header &header, const bool) + { + const size_t bin_count = (header.bin_count() == 0) ? static_cast(q_metric::MAX_Q_BINS) : header.bin_count(); + const std::streamsize count = stream_map(stream, metric.m_qscore_hist, bin_count); + resize_accumulated(stream, metric); + return count; + } + + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const q_metric::header_type &header) + { + const size_t bin_count = header.bin_count() == 0 ? static_cast(q_metric::MAX_Q_BINS) : header.bin_count(); + return static_cast(sizeof(metric_id_t) + sizeof(count_t) * bin_count); + } + + /** Map reading/writing a header to a stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param header source/destination header + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream_for_header(Stream &stream, Header &header) + { + bool_t has_bins = header.bin_count() > 0; + std::streamsize count = 0; + count += stream_map(stream, has_bins); + if (stream.fail()) return count; + if (!has_bins) return count; + bin_count_t bin_count = static_cast(header.bin_count()); + count += stream_map(stream, bin_count); + if (stream.fail()) return count; + if(bin_count==0) + INTEROP_THROW(bad_format_exception, "Zero bins is not supported"); + INTEROP_ASSERT(bin_count > 0); + bin_t tmp[q_metric::MAX_Q_BINS]; + map_resize(header.m_qscore_bins, bin_count); + + copy_lower_write(header.m_qscore_bins, tmp); + count += stream_map(stream, tmp, bin_count); + copy_lower_read(header.m_qscore_bins, tmp); + + copy_upper_write(header.m_qscore_bins, tmp); + count += stream_map(stream, tmp, bin_count); + copy_upper_read(header.m_qscore_bins, tmp); + + copy_value_write(header.m_qscore_bins, tmp); + count += stream_map(stream, tmp, bin_count); + copy_value_read(header.m_qscore_bins, tmp); + + return count; + } + + /** Compute header size + * + * @param header q-metric header + * @return header size + */ + static record_size_t compute_header_size(const q_metric::header_type &header) + { + if (header.bin_count() == 0) + return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); + return static_cast(sizeof(record_size_t) + + sizeof(version_t) + // version + sizeof(bool_t) + // has bins + sizeof(bin_count_t) + // number of bins + header.bin_count() * 3 * sizeof(bin_t)); + } + + private: + template + static void copy_lower_read(std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) bins[i].m_lower = tmp[i]; + } + + static void copy_lower_read(const std::vector &, bin_t *) + { } + + template + static void copy_lower_write(const std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) tmp[i] = static_cast< bin_t >(bins[i].m_lower); + } + + static void copy_lower_write(std::vector &, bin_t *) + { } + + private: + template + static void copy_upper_read(std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) bins[i].m_upper = tmp[i]; + } + + static void copy_upper_read(const std::vector &, bin_t *) + { } + + template + static void copy_upper_write(const std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) tmp[i] = static_cast< bin_t >(bins[i].m_upper); + } + + static void copy_upper_write(std::vector &, bin_t *) + { } + + private: + template + static void copy_value_read(std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) bins[i].m_value = tmp[i]; + } + + static void copy_value_read(const std::vector &, bin_t *) + { } + + template + static void copy_value_write(const std::vector &bins, bin_t (&tmp)[N]) + { + INTEROP_ASSERT(bins.size() <= N); + for (size_t i = 0; i < bins.size(); ++i) tmp[i] = static_cast< bin_t >(bins[i].m_value); + } + + static void copy_value_write(std::vector &, bin_t *) + { } + + private: + static void resize_accumulated(const char*, q_metric& metric) + { + metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); + } + static void resize_accumulated(std::istream &, q_metric &metric) + { + metric.m_qscore_hist_cumulative.resize(metric.m_qscore_hist.size(), 0); + } + + static void resize_accumulated(std::ostream &, const q_metric &) + { } + }; + + /** Packed layout of a q-score bin + * + */ + struct q_score_bin_layout + { + /** Bin type */ + typedef ::uint8_t bin_t; + /** lower bound of bin */ + bin_t m_lower; + /** upper bound of bin */ + bin_t m_upper; + /** value of bin */ + bin_t m_value; + + /** Copy constructor + * + * @param bin q-score bin + */ + q_score_bin_layout(const q_score_bin &bin = q_score_bin()) + { + m_lower = static_cast< bin_t >(bin.lower()); + m_upper = static_cast< bin_t >(bin.upper()); + m_value = static_cast< bin_t >(bin.value()); + } + + /** Assign a point2d to this layout + * + * @param bin q-score bin + * @return this layout + */ + q_score_bin_layout &operator=(const q_score_bin &bin) + { + m_lower = static_cast< bin_t >(bin.lower()); + m_upper = static_cast< bin_t >(bin.upper()); + m_value = static_cast< bin_t >(bin.value()); + return *this; + } + + }; + + /** Q-score by lane Metric Record Layout Version 4 + * + * This class provides an interface to reading the q_by_lane_metric file: + * - InterOp/QMetricsByLane.bin + * - InterOp/QMetricsByLaneOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: q_by_lane_metric + * 2. Version: 4 + */ + template<> + struct generic_layout : public generic_layout + { + /** Skip inserting this metric into the metric set + * + * This function was originally added to skip control records in tile metrics. + * + * @param metric metric to check + * @return true, if the metric id is 0 + */ + template + static bool is_valid(const LayoutId& id) + { + return id.lane > 0; + } + }; + /** Q-score by lane Metric Record Layout Version 5 + * + * This class provides an interface to reading the q_by_lane_metric file: + * - InterOp/QMetricsByLane.bin + * - InterOp/QMetricsByLaneOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: q_by_lane_metric + * 2. Version: 5 + */ + template<> + struct generic_layout : public generic_layout + { + /** Skip inserting this metric into the metric set + * + * This function was originally added to skip control records in tile metrics. + * + * @param metric metric to check + * @return true, if the metric id is 0 + */ + template + static bool is_valid(const LayoutId& id) + { + return id.lane > 0; + } + }; + /** Q-score by lane Metric Record Layout Version 6 + * + * This class provides an interface to reading the q_by_lane_metric file: + * - InterOp/QMetricsByLane.bin + * - InterOp/QMetricsByLaneOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: q_by_lane_metric + * 2. Version: 6 + */ + template<> + struct generic_layout : public generic_layout + { + /** Skip inserting this metric into the metric set + * + * This function was originally added to skip control records in tile metrics. + * + * @param metric metric to check + * @return true, if the metric id is 0 + */ + template + static bool is_valid(const LayoutId& id) + { + return id.lane > 0; + } + }; #pragma pack()// DO NOT MOVE - }}} +}}} INTEROP_FORCE_LINK_DEF(q_metric) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 4 ) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 5 ) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 6 ) + +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 4) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 5) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 6) INTEROP_FORCE_LINK_DEF(q_by_lane_metric) -INTEROP_REGISTER_METRIC_ANOTHER_GENERIC_LAYOUT(q_metric, q_by_lane_metric, 4) -INTEROP_REGISTER_METRIC_ANOTHER_GENERIC_LAYOUT(q_metric, q_by_lane_metric, 5) -INTEROP_REGISTER_METRIC_ANOTHER_GENERIC_LAYOUT(q_metric, q_by_lane_metric, 6) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_by_lane_metric, 4) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_by_lane_metric, 5) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_by_lane_metric, 6) diff --git a/src/interop/model/metrics/tile_metric.cpp b/src/interop/model/metrics/tile_metric.cpp index 5295ca821..a4952dec8 100644 --- a/src/interop/model/metrics/tile_metric.cpp +++ b/src/interop/model/metrics/tile_metric.cpp @@ -8,9 +8,9 @@ * @copyright GNU Public License. */ +#include "interop/util/math.h" #include "interop/model/metrics/tile_metric.h" #include "interop/io/format/metric_format_factory.h" -#include "interop/util/math.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" @@ -18,233 +18,256 @@ using namespace illumina::interop::model::metrics; namespace illumina { namespace interop { namespace io { + #pragma pack(1) - /** Tile Metric Record Layout Version 2 - * - * This class provides an interface to reading the tile metric file: - * - InterOp/TileMetrics.bin - * - InterOp/TileMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: tile_metric - * 2. Version: 2 - */ - template<> - struct generic_layout : public default_layout<2, 1 /*Multi record */> + + /** Tile Metric Record Layout Version 2 + * + * This class provides an interface to reading the tile metric file: + * - InterOp/TileMetrics.bin + * - InterOp/TileMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: tile_metric + * 2. Version: 2 + */ + template<> + struct generic_layout : public default_layout<2, 1 /*Multi record */> + { + /** @page tile_v2 Tile Version 2 + * + * This class provides an interface to reading the tile metric file: + * - InterOp/TileMetrics.bin + * - InterOp/TileMetricsOut.bin + * + * The file format for tile metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (2) + * byte 1: record size (10) + * + * @b n-Records + * + * illumina::interop::io::layout::base_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 2 bytes: code (uint16) + * 4 bytes: value (float32) + */ + /** Metric ID type */ + typedef layout::base_metric< ::uint16_t > metric_id_t; + /** Record type */ + typedef generic_layout record_t; + /** Code type */ + typedef ::uint16_t code_t; + /** Value type */ + typedef float value_t; + /** Code for each tile metric + */ + enum TileMetricCode + { + ClusterDensity = 100, + ClusterDensityPf = 101, + ClusterCount = 102, + ClusterCountPf = 103, + Phasing = 200, + Prephasing = 201, + PercentAligned = 300, + ControlLane = 400 + }; + /** Tile Metric Code */ + code_t code; + /** Tile Metric Value */ + value_t value; + + /** Read metric from the input stream + * + * @param stream input stream + * @param metric destination metric + * @param is_new true if metric is new + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(std::istream &stream, Metric &metric, Header &, const bool is_new) + { + record_t rec; + std::streamsize count = stream_map(stream, rec); + if (stream.fail()) return count; + float val = rec.value; + if (val != val) val = 0; // TODO: Remove this after baseline + switch (rec.code) { - /** @page tile_v2 Tile Version 2 - * - * This class provides an interface to reading the tile metric file: - * - InterOp/TileMetrics.bin - * - InterOp/TileMetricsOut.bin - * - * The file format for tile metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number - * byte 1: record size - * - * @b n-Records - * - * illumina::interop::io::layout::base_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 2 bytes: code (uint16) - * 4 bytes: value (float32) - */ - /** Metric ID type */ - typedef layout::base_metric metric_id_t; - /** Record type */ - typedef generic_layout record_t; - /** Code type */ - typedef ::uint16_t code_t; - /** Value type */ - typedef float value_t; - /** Code for each tile metric - */ - enum TileMetricCode - { - ClusterDensity = 100, - ClusterDensityPf = 101, - ClusterCount = 102, - ClusterCountPf = 103, - Phasing = 200, - Prephasing = 201, - PercentAligned = 300, - ControlLane = 400 - }; - /** Tile Metric Code */ - code_t code; - /** Tile Metric Value */ - value_t value; - - /** Read metric from the input stream - * - * @param stream input stream - * @param metric destination metric - * @param is_new true if metric is new - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(std::istream& stream, Metric& metric, Header&, const bool is_new) - { - record_t rec; - std::streamsize count = stream_map< record_t >(stream, rec); - if(stream.fail()) return count; - float val = rec.value; - if( val != val ) val = 0; // TODO: Remove this after baseline - switch(rec.code) - { - case ControlLane: - if(is_new) metric.set_base(metric_id_t()); - break; - case ClusterDensity: - metric.m_cluster_density = val; - break; - case ClusterDensityPf: - metric.m_cluster_density_pf = val; - break; - case ClusterCount: - metric.m_cluster_count = val; - break; - case ClusterCountPf: - metric.m_cluster_count_pf = val; - break; - default: - if( rec.code % Phasing < 100 ) - { - //code = Prephasing+read*2; - int code_offset = rec.code % Phasing; - if(code_offset%2 == 0) - { - get_read(metric, (code_offset/2)+1)->percent_phasing(val*100); - } - else - { - get_read(metric, (code_offset+1)/2)->percent_prephasing(val*100); - } - } - else if(rec.code % PercentAligned < 100) - { - int code_offset = rec.code % PercentAligned; - get_read(metric, code_offset+1)->percent_aligned(val); - } - else INTEROP_THROW(bad_format_exception, "Unexpected tile code"); - }; - - return count; - } - template - static std::streamsize map_stream(const char*, Metric&, Header&, const bool) - { - INTEROP_THROW(std::runtime_error, "This function should not be called"); - } - /** Write metric to the output stream - * - * @param out output stream - * @param metric source metric - * @return number of bytes read or total number of bytes written - */ - template - static std::streamsize map_stream(std::ostream& out, Metric& metric, Header&, const bool) - { - record_t rec; - metric_id_t metric_id; - metric_id.set(metric); - bool write_id = false; - - // We always write this out, even if it is NaN - // We always write out ID for the first record - if(!std::isnan(metric.m_cluster_density)) - { - rec.value = metric.m_cluster_density; - rec.code = ClusterDensity; - if(write_id) write_binary(out, metric_id); - write_id=true; - write_binary(out, rec); - } - if(!std::isnan(metric.m_cluster_density_pf)) - { - rec.value = metric.m_cluster_density_pf; - rec.code = ClusterDensityPf; - if(write_id) write_binary(out, metric_id); - write_id=true; - write_binary(out, rec); - } - if(!std::isnan(metric.m_cluster_count)) - { - rec.value = metric.m_cluster_count; - rec.code = ClusterCount; - if(write_id) write_binary(out, metric_id); - write_id=true; - write_binary(out, rec); - } - if(!std::isnan(metric.m_cluster_count_pf)) + case ControlLane: + if (is_new) metric.set_base(metric_id_t()); + break; + case ClusterDensity: + metric.m_cluster_density = val; + break; + case ClusterDensityPf: + metric.m_cluster_density_pf = val; + break; + case ClusterCount: + metric.m_cluster_count = val; + break; + case ClusterCountPf: + metric.m_cluster_count_pf = val; + break; + default: + if (rec.code % Phasing < 100) { - rec.value = metric.m_cluster_count_pf; - rec.code = ClusterCountPf; - if(write_id) write_binary(out, metric_id); - write_id=true; - write_binary(out, rec); + //code = Prephasing+read*2; + int code_offset = rec.code % Phasing; + if (code_offset % 2 == 0) + { + get_read(metric, (code_offset / 2) + 1)->percent_phasing(val * 100); + } + else + { + get_read(metric, (code_offset + 1) / 2)->percent_prephasing(val * 100); + } } - typedef tile_metric::read_metric_vector::const_iterator const_iterator; - for(const_iterator beg = metric.read_metrics().begin();beg != metric.read_metrics().end();beg++) + else if (rec.code % PercentAligned < 100) { - const int read = beg->read()-1; - rec.code = static_cast< code_t >(Prephasing+read*2); - rec.value = beg->percent_prephasing(); - if(write_id) write_binary(out, metric_id); - write_id=true; - write_binary(out, rec); - rec.code = static_cast< code_t >(Phasing+read*2); - rec.value = beg->percent_phasing(); - write_binary(out, metric_id); - write_binary(out, rec); - rec.code = static_cast< code_t >(PercentAligned+read); - rec.value = beg->percent_aligned(); - write_binary(out, metric_id); - write_binary(out, rec); + int code_offset = rec.code % PercentAligned; + get_read(metric, code_offset + 1)->percent_aligned(val); } - return out.tellp(); - } - /** Compute the size of a single metric record - * - * @return record size - */ - static record_size_t compute_size(const tile_metric::header_type&) + else + INTEROP_THROW(bad_format_exception, "Unexpected tile code"); + }; + + return count; + } + + /** Write metric to the output stream + * + * @param out output stream + * @param metric source metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(std::ostream &out, Metric &metric, Header &, const bool) + { + record_t rec; + metric_id_t metric_id; + metric_id.set(metric); + bool write_id = false; + + // We always write this out, even if it is NaN + // We always write out ID for the first record + if (!std::isnan(metric.m_cluster_density)) + { + rec.value = metric.m_cluster_density; + rec.code = ClusterDensity; + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); + } + if (!std::isnan(metric.m_cluster_density_pf)) + { + rec.value = metric.m_cluster_density_pf; + rec.code = ClusterDensityPf; + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); + } + if (!std::isnan(metric.m_cluster_count)) + { + rec.value = metric.m_cluster_count; + rec.code = ClusterCount; + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); + } + if (!std::isnan(metric.m_cluster_count_pf)) + { + rec.value = metric.m_cluster_count_pf; + rec.code = ClusterCountPf; + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); + } + typedef tile_metric::read_metric_vector::const_iterator const_iterator; + for (const_iterator beg = metric.read_metrics().begin(); beg != metric.read_metrics().end(); beg++) + { + const int read = beg->read() - 1; + if(!std::isnan(beg->percent_prephasing())) { - return static_cast< record_size_t >(sizeof(metric_id_t)+sizeof(record_t)); + rec.code = static_cast< code_t >(Prephasing + read * 2); + rec.value = beg->percent_prephasing(); + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); } - /** Compute header size - * - * @return header size - */ - static record_size_t compute_header_size(const tile_metric::header_type&) + if(!std::isnan(beg->percent_phasing())) { - return static_cast(sizeof(record_size_t) + sizeof(version_t)); + rec.code = static_cast< code_t >(Phasing + read * 2); + rec.value = beg->percent_phasing(); + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); } - - private: - static tile_metric::read_metric_vector::iterator get_read(tile_metric& metric, tile_metric::read_metric_type::uint_t read) + if(!std::isnan(beg->percent_aligned())) { - tile_metric::read_metric_vector::iterator it = metric.m_read_metrics.begin(); - for(;it != metric.m_read_metrics.end();it++) - if(it->read() == read) return it; - metric.m_read_metrics.push_back(model::metrics::read_metric(read)); - return metric.m_read_metrics.begin()+metric.m_read_metrics.size()-1; + rec.code = static_cast< code_t >(PercentAligned + read); + rec.value = beg->percent_aligned(); + if (write_id) write_binary(out, metric_id); + write_id = true; + write_binary(out, rec); } - }; + } + return out.tellp(); + } + /** Throws an unimplemented error + */ + template + static std::streamsize map_stream(const char*, const Metric &, const Header &, const bool) + { + INTEROP_THROW(std::runtime_error, "Function not implemented"); + } + + /** Compute the size of a single metric record + * + * @return record size + */ + static record_size_t compute_size(const tile_metric::header_type &) + { + return static_cast< record_size_t >(sizeof(metric_id_t) + sizeof(record_t)); + } + + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const tile_metric::header_type &) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + + private: + static tile_metric::read_metric_vector::iterator get_read(tile_metric &metric, + tile_metric::read_metric_type::uint_t read) + { + tile_metric::read_metric_vector::iterator it = metric.m_read_metrics.begin(); + for (; it != metric.m_read_metrics.end(); it++) + if (it->read() == read) return it; + metric.m_read_metrics.push_back(model::metrics::read_metric(read)); + return metric.m_read_metrics.begin() + metric.m_read_metrics.size() - 1; + } + }; + + + #pragma pack() }}} INTEROP_FORCE_LINK_DEF(tile_metric) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(tile_metric, 2 ) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(tile_metric, 2) diff --git a/src/interop/model/run/info.cpp b/src/interop/model/run/info.cpp index 6ed908844..82a65fee3 100644 --- a/src/interop/model/run/info.cpp +++ b/src/interop/model/run/info.cpp @@ -169,12 +169,12 @@ namespace illumina { namespace interop { namespace model { namespace run xml::missing_xml_element_exception, xml::xml_parse_exception) { - if (run_folder.find("RunInfo.xml") != std::string::npos) + if (run_folder.find(io::paths::run_info()) != std::string::npos) { read_file(run_folder); return; } - read_file(io::combine(run_folder, "RunInfo.xml")); + read_file(io::paths::run_info(run_folder)); } /** Test if tile list matches flowcell layout * diff --git a/src/interop/model/run/parameters.cpp b/src/interop/model/run/parameters.cpp index 47c7d1220..0eea1e4d0 100644 --- a/src/interop/model/run/parameters.cpp +++ b/src/interop/model/run/parameters.cpp @@ -117,19 +117,19 @@ namespace illumina { namespace interop { namespace model { namespace run xml::xml_parse_exception) { - if (run_folder.find("RunParameters.xml") != std::string::npos || - run_folder.find("runParameters.xml") != std::string::npos) + if (run_folder.find(io::paths::run_parameters()) != std::string::npos || + run_folder.find(io::paths::run_parameters(true)) != std::string::npos) { read_file(run_folder); return; } try { - read_file(io::combine(run_folder, "runParameters.xml")); + read_file(io::paths::run_parameters(run_folder, true)); } catch (const xml_file_not_found_exception &) { - read_file(io::combine(run_folder, "RunParameters.xml")); + read_file(io::paths::run_parameters(run_folder)); } } diff --git a/src/interop/model/run_metrics.cpp b/src/interop/model/run_metrics.cpp index 50fa1c1cf..57dad8772 100644 --- a/src/interop/model/run_metrics.cpp +++ b/src/interop/model/run_metrics.cpp @@ -64,7 +64,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics for (typename MetricSet::const_iterator it = metrics.begin(); it != metrics.end(); ++it) { INTEROP_ASSERTMSG(it->tile() > 0, it->lane() << "_" << it->tile() << " @ " << it->cycle()); - m_map[it->id()] = *it; + m_map[it->cycle_hash()] = *it; } } @@ -93,18 +93,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics bool m_empty; }; - - struct check_for_each_data_source - { - check_for_each_data_source(const std::string &f) : m_run_folder(f) - {} - template - void operator()(MetricSet &metrics)const - { - metrics.data_source_exists(io::interop_exists(m_run_folder, metrics)); - } - std::string m_run_folder; - }; struct clear_metric { template @@ -118,7 +106,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics { typedef const unsigned char* bool_pointer; read_func(const std::string &f, bool_pointer load_metric_check=0) : - m_run_folder(f), m_load_metric_check(load_metric_check) + m_run_folder(f), m_load_metric_check(load_metric_check), m_are_all_files_missing(true) {} template @@ -127,6 +115,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics // If the m_load_metric_check is not set, read in the metric // Otherwise, check if the metric should be read and that it is not empty // This logic is for SAV OnDemand (TM) loading + const bool is_index_metrics = static_cast(MetricSet::TYPE) == constants::Index; if(m_load_metric_check != 0 && (m_load_metric_check[MetricSet::TYPE] == 0 || !metrics.empty())) { return 0; @@ -134,26 +123,28 @@ namespace illumina { namespace interop { namespace model { namespace metrics try { io::read_interop(m_run_folder, metrics); + if(m_are_all_files_missing && !is_index_metrics) m_are_all_files_missing=false; } catch (const io::file_not_found_exception &) { - try - { - io::read_interop(m_run_folder, metrics, - false /** Search for XMetrics.bin not XMetricsOut.bin */); - } - catch (const io::file_not_found_exception &) - { return 1; } - catch (const io::incomplete_file_exception &) - { return 2; } + return 1; } catch (const io::incomplete_file_exception &) - { return 2; } + { + if(m_are_all_files_missing && !is_index_metrics)m_are_all_files_missing=false; + return 2; + } return 0; } + bool are_all_files_missing()const + { + return m_are_all_files_missing; + } + std::string m_run_folder; bool_pointer m_load_metric_check; + mutable bool m_are_all_files_missing; }; struct write_func @@ -301,26 +292,39 @@ namespace illumina { namespace interop { namespace model { namespace metrics constants::tile_naming_method m_naming_method; }; + struct check_for_each_data_source + { + check_for_each_data_source(const std::string &f) : m_run_folder(f) + {} + template + void operator()(MetricSet &metrics)const + { + metrics.data_source_exists(io::interop_exists(m_run_folder, metrics)); + } + std::string m_run_folder; + }; + /** Read binary metrics and XML files from the run folder * * @param run_folder run folder path */ void run_metrics::read(const std::string &run_folder) - throw(xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception, - io::file_not_found_exception, - io::bad_format_exception, - io::incomplete_file_exception, - io::format_exception, - model::index_out_of_bounds_exception, - model::invalid_tile_naming_method, - model::invalid_run_info_exception) + throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception, + io::file_not_found_exception, + io::bad_format_exception, + io::incomplete_file_exception, + io::format_exception, + model::index_out_of_bounds_exception, + model::invalid_tile_naming_method, + model::invalid_run_info_exception) { - read_metrics(run_folder); + clear(); const size_t count = read_xml(run_folder); + read_metrics(run_folder); finalize_after_load(count); } /** Read binary metrics and XML files from the run folder @@ -343,8 +347,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics model::invalid_run_info_exception, invalid_parameter) { - read_metrics(run_folder, valid_to_load); + clear(); const size_t count = read_xml(run_folder); + read_metrics(run_folder, valid_to_load); finalize_after_load(count); check_for_data_sources(run_folder); } @@ -390,7 +395,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics xml::missing_xml_element_exception, xml::xml_parse_exception) { - const size_t count = logic::metric::count_legacy_q_score_bins(get_set()); + const size_t count = logic::metric::count_legacy_q_score_bins(get()); if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count)) { @@ -416,9 +421,10 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param count number of bins for legacy q-metrics */ void run_metrics::finalize_after_load(size_t count) - throw(io::format_exception, - model::invalid_tile_naming_method, - model::index_out_of_bounds_exception, model::invalid_run_info_exception) + throw(io::format_exception, + model::invalid_tile_naming_method, + model::index_out_of_bounds_exception, + model::invalid_run_info_exception) { if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) { @@ -428,22 +434,27 @@ namespace illumina { namespace interop { namespace model { namespace metrics } if (count == std::numeric_limits::max()) { - if (get_set().size() > 0) - count = logic::metric::count_legacy_q_score_bins(get_set()); - else if (get_set().size()) - count = logic::metric::count_legacy_q_score_bins(get_set()); + if (get().size() > 0) + count = logic::metric::count_legacy_q_score_bins(get()); + else if (get().size()) + count = logic::metric::count_legacy_q_score_bins(get()); } - logic::metric::populate_legacy_q_score_bins(get_set().bins(), m_run_parameters.instrument_type(), + logic::metric::populate_legacy_q_score_bins(get().bins(), m_run_parameters.instrument_type(), count); - if (get_set().size() > 0 && get_set().size() == 0) - logic::metric::create_collapse_q_metrics(get_set(), get_set()); - if (get_set().size() > 0 && get_set().size() == 0) - logic::metric::create_q_metrics_by_lane(get_set(), get_set()); - logic::metric::populate_cumulative_distribution(get_set()); - logic::metric::populate_cumulative_distribution(get_set()); - logic::metric::populate_cumulative_distribution(get_set()); - INTEROP_ASSERT( - get_set().size() == 0 || get_set().size() == get_set().size()); + if (get().size() > 0 && get().size() == 0) + { + logic::metric::create_collapse_q_metrics(get(), get()); + } + INTEROP_ASSERTMSG( + get().size() == 0 || + get().size() == get().size(), + get().size() << " == " << get().size()); + if (get().size() > 0 && get().size() == 0) + logic::metric::create_q_metrics_by_lane(get(), get()); + logic::metric::populate_cumulative_distribution(get()); + logic::metric::populate_cumulative_distribution(get()); + logic::metric::populate_cumulative_distribution(get()); + if (m_run_info.channels().empty()) { legacy_channel_update(m_run_parameters.instrument_type()); @@ -453,13 +464,13 @@ namespace illumina { namespace interop { namespace model { namespace metrics } if (!empty()) { - if (run_info().flowcell().naming_method() == constants::UnknownTileNamingMethod) - INTEROP_THROW(model::invalid_tile_naming_method, - "Unknown tile naming method - update your RunInfo.xml"); + if(run_info().flowcell().naming_method() == constants::UnknownTileNamingMethod) + INTEROP_THROW(model::invalid_tile_naming_method, "Unknown tile naming method - update your RunInfo.xml"); m_run_info.validate(); validate(); } - extraction_metric_set_t& extraction_metrics = get_set(); + typedef metric_base::metric_set< extraction_metric > extraction_metric_set_t; + extraction_metric_set_t &extraction_metrics = get(); // Trim excess channel data for imaging table for (extraction_metric_set_t::iterator it = extraction_metrics.begin(); it != extraction_metrics.end(); ++it) it->trim(run_info().channels().size()); @@ -575,7 +586,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics } /** Check if the metric group is empty * - * @param group_id prefix of interop group metric id + * @param group_id id of interop group metric * @return true if metric is empty */ bool run_metrics::is_group_empty(const constants::metric_group group_id) const diff --git a/src/interop/model/summary/index_summary.cpp b/src/interop/model/summary/index_summary.cpp new file mode 100644 index 000000000..62501f15d --- /dev/null +++ b/src/interop/model/summary/index_summary.cpp @@ -0,0 +1,152 @@ +/** Register format layout for index_flowcell_summary + * + * Each version of the image metrics file has a layout defined below. + * + * @file + * @date 6/24/16. + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include +#include "interop/model/summary/index_flowcell_summary.h" +#include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/default_layout.h" + +using namespace illumina::interop::model::summary; +using namespace illumina::interop; + +namespace illumina{ namespace interop{ namespace io { + + /** Index summary Record Layout Version 1 + * + * This class provides an interface to reading the image metric file: + * - InterOp/IndexSummary.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: index_flowcell_summary + * 2. Version: 1 + */ + template<> + struct generic_layout : public default_layout<1> + { + /** @page summary_v1 Index Summary v1 + * + * This class provides an interface to reading the image metric file: + * - InterOp/IndexSummary.bin + * + * The file format for index_flowcell_summary is as follows: + * + */ + + /** Map reading/writing a summary to a stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param summary source/destination summary + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_records(Stream& stream, Summary& summary) + { + std::streamsize count = 0; + count += map_lane_summary(stream, summary.begin(), summary.end()); + return count; + } + + /** Map reading/writing a summary to a stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param header source/destination header + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_header(Stream& stream, Summary& header) + { + std::streamsize count = 0; + size_t lane_count = header.size(); + count += stream_map< size_t >(stream, lane_count); + map_resize(header, lane_count); + return count; + } + private: + template + static std::streamsize map_lane_summary(Stream& stream, + LaneSummaryIterator lane_beg, + LaneSummaryIterator lane_end) + { + typedef index_lane_summary::read_count_t read_count_t; + std::streamsize count = 0; + for(;lane_beg != lane_end;++lane_beg) + { + count += stream_map< read_count_t >(stream, lane_beg->m_total_reads); + count += stream_map< read_count_t >(stream, lane_beg->m_total_pf_reads); + count += stream_map< float >(stream, lane_beg->m_total_fraction_mapped_reads); + count += stream_map< float >(stream, lane_beg->m_mapped_reads_cv); + count += stream_map< float >(stream, lane_beg->m_min_mapped_reads); + count += stream_map< float >(stream, lane_beg->m_max_mapped_reads); + + size_t index_count = lane_beg->size(); + count += stream_map< size_t >(stream, index_count); + map_resize(*lane_beg, index_count); + map_count_summary(stream, lane_beg->begin(), lane_beg->end()); + } + return count; + } + template + static std::streamsize map_count_summary(Stream& stream, + IndexCountIterator count_beg, + IndexCountIterator count_end) + { + std::streamsize count = 0; + for(;count_beg != count_end;++count_beg) + { + count += stream_map< size_t >(stream, count_beg->m_id); + count += stream_map(stream, count_beg->m_index1); + count += stream_map(stream, count_beg->m_index2); + count += stream_map< float >(stream, count_beg->m_fraction_mapped); + count += stream_map< ::uint64_t >(stream, count_beg->m_cluster_count); + count += stream_map(stream, count_beg->m_sample_id); + count += stream_map(stream, count_beg->m_project_name); + } + return count; + } + }; + +}}} +namespace illumina { namespace interop { namespace model { namespace summary +{ + std::ostream &operator<<(std::ostream &out, const index_flowcell_summary &summary) + { + typedef io::generic_layout layout_t; + typedef layout_t::version_t version_t; + const version_t version = static_cast(layout_t::VERSION); + io::write_binary(out, version); + if (!out.good()) INTEROP_THROW(io::bad_format_exception, "Cannot write to disk"); + layout_t::map_header(out, summary); + if (!out.good()) INTEROP_THROW(io::bad_format_exception, "Cannot write to disk"); + layout_t::map_records(out, summary); + if (!out.good()) INTEROP_THROW(io::bad_format_exception, "Cannot write to disk"); + return out; + } + + std::istream &operator>>(std::istream &in, index_flowcell_summary &summary) + { + typedef io::generic_layout layout_t; + + + if (!in.good()) INTEROP_THROW(io::incomplete_file_exception, "Empty file found"); + const int version = in.get(); + if (version == -1) INTEROP_THROW(io::incomplete_file_exception, "Empty file found"); + if (version != layout_t::VERSION) INTEROP_THROW(io::bad_format_exception, "Version mismatch"); + + layout_t::map_header(in, summary); + layout_t::map_records(in, summary); + return in; + } +}}}} + diff --git a/src/interop/model/summary/run_summary.cpp b/src/interop/model/summary/run_summary.cpp index c1df987ae..108f610a5 100644 --- a/src/interop/model/summary/run_summary.cpp +++ b/src/interop/model/summary/run_summary.cpp @@ -52,10 +52,11 @@ namespace illumina{ namespace interop{ namespace io { static std::streamsize map_records(Stream& stream, Summary& summary) { std::streamsize count = 0; + count += map_cycle_state(stream, summary.cycle_state()); - count += map_metric_summary(stream, summary.nonindex_summary()); - count += map_metric_summary(stream, summary.total_summary()); - count += map_read_summary(stream, summary.begin(), summary.end()); + count += map_metric_summary(stream, summary.nonindex_summary(), summary.channel_count()); + count += map_metric_summary(stream, summary.total_summary(), summary.channel_count()); + count += map_read_summary(stream, summary.begin(), summary.end(), summary.channel_count()); return count; } @@ -71,54 +72,89 @@ namespace illumina{ namespace interop{ namespace io { static std::streamsize map_header(Stream& stream, Header& header) { std::streamsize count = 0; + count += stream_map< size_t >(stream, header.m_surface_count); count += stream_map< size_t >(stream, header.m_read_count); count += stream_map< size_t >(stream, header.m_lane_count); + count += stream_map< ::uint8_t >(stream, header.m_channel_count); return count; } private: template - static std::streamsize map_read_summary(Stream& stream, ReadSummaryIterator read_beg, ReadSummaryIterator read_end) + static std::streamsize map_read_summary(Stream& stream, + ReadSummaryIterator read_beg, + ReadSummaryIterator read_end, + const size_t channel_count) { std::streamsize count = 0; for(;read_beg != read_end;++read_beg) { count += map_read(stream, read_beg->m_read); - count += map_metric_summary(stream, read_beg->m_metric_summary); - count += map_lane_summary(stream, read_beg->begin(), read_beg->end()); + count += map_metric_summary(stream, read_beg->m_metric_summary, channel_count); + count += map_lane_summary(stream, read_beg->begin(), read_beg->end(), channel_count); } return count; } template - static std::streamsize map_lane_summary(Stream& stream, LaneSummaryIterator lane_beg, LaneSummaryIterator lane_end) + static std::streamsize map_lane_summary(Stream& stream, + LaneSummaryIterator lane_beg, + LaneSummaryIterator lane_end, + const size_t channel_count) { std::streamsize count = 0; for(;lane_beg != lane_end;++lane_beg) { count += stream_map< size_t >(stream, lane_beg->m_lane); count += stream_map< size_t >(stream, lane_beg->m_tile_count); - count += stream_map< float >(stream, lane_beg->m_percent_gt_q30); - count += stream_map< float >(stream, lane_beg->m_yield_g); - count += stream_map< float >(stream, lane_beg->m_projected_yield_g); - count += stream_map< float >(stream, lane_beg->m_reads); - count += stream_map< float >(stream, lane_beg->m_reads_pf); - count += map_metric_stat(stream, lane_beg->m_density); - count += map_metric_stat(stream, lane_beg->m_density_pf); - count += map_metric_stat(stream, lane_beg->m_cluster_count); - count += map_metric_stat(stream, lane_beg->m_cluster_count_pf); - count += map_metric_stat(stream, lane_beg->m_percent_pf); - count += map_metric_stat(stream, lane_beg->m_phasing); - count += map_metric_stat(stream, lane_beg->m_prephasing); - count += map_metric_stat(stream, lane_beg->m_percent_aligned); - count += map_metric_stat(stream, lane_beg->m_error_rate); - count += map_metric_stat(stream, lane_beg->m_error_rate_35); - count += map_metric_stat(stream, lane_beg->m_error_rate_50); - count += map_metric_stat(stream, lane_beg->m_error_rate_75); - count += map_metric_stat(stream, lane_beg->m_error_rate_100); - count += map_metric_stat(stream, lane_beg->m_first_cycle_intensity); + count += map_summary_stat(stream, *lane_beg, channel_count); count += map_cycle_state(stream, lane_beg->m_cycle_state); + count += map_surface_summary(stream, lane_beg->begin(), lane_beg->end(), channel_count); + } + return count; + } + template + static std::streamsize map_surface_summary(Stream& stream, + SurfaceSummaryIterator surface_beg, + SurfaceSummaryIterator surface_end, + const size_t channel_count) + { + std::streamsize count = 0; + for(;surface_beg != surface_end;++surface_beg) + { + count += stream_map< size_t >(stream, surface_beg->m_surface); + count += stream_map< size_t >(stream, surface_beg->m_tile_count); + count += map_summary_stat(stream, *surface_beg, channel_count); } return count; } + template + static std::streamsize map_summary_stat(Stream& stream, SummaryStat& stat, const size_t /*channel_count*/) + { + std::streamsize count = 0; + + count += stream_map< float >(stream, stat.m_percent_gt_q30); + count += stream_map< float >(stream, stat.m_yield_g); + count += stream_map< float >(stream, stat.m_projected_yield_g); + count += stream_map< float >(stream, stat.m_reads); + count += stream_map< float >(stream, stat.m_reads_pf); + + count += map_metric_stat(stream, stat.m_density); + count += map_metric_stat(stream, stat.m_density_pf); + count += map_metric_stat(stream, stat.m_cluster_count); + count += map_metric_stat(stream, stat.m_cluster_count_pf); + count += map_metric_stat(stream, stat.m_percent_pf); + count += map_metric_stat(stream, stat.m_phasing); + count += map_metric_stat(stream, stat.m_prephasing); + count += map_metric_stat(stream, stat.m_percent_aligned); + count += map_metric_stat(stream, stat.m_error_rate); + count += map_metric_stat(stream, stat.m_error_rate_35); + count += map_metric_stat(stream, stat.m_error_rate_50); + count += map_metric_stat(stream, stat.m_error_rate_75); + count += map_metric_stat(stream, stat.m_error_rate_100); + count += map_metric_stat(stream, stat.m_first_cycle_intensity); + + return count; + } + template static std::streamsize map_metric_stat(Stream& stream, MetricStat& stat) { @@ -156,7 +192,9 @@ namespace illumina{ namespace interop{ namespace io { return count; } template - static std::streamsize map_metric_summary(Stream& stream, MetricSummary& metric_summary) + static std::streamsize map_metric_summary(Stream& stream, + MetricSummary& metric_summary, + const size_t /*channel_count*/) { std::streamsize count = 0; diff --git a/src/interop/util/filesystem.cpp b/src/interop/util/filesystem.cpp index 67f795c70..d791d4ef4 100644 --- a/src/interop/util/filesystem.cpp +++ b/src/interop/util/filesystem.cpp @@ -42,6 +42,20 @@ namespace illumina { namespace interop { namespace io } return path + name; } + /** Combine two directories or a directory and a filename into a file path + * + * This function provides a platform independent way to generate a file path. It currently supports most + * operating systems include Mac OSX, Windows and Linux/Unix. + * + * @param path1 string representing a file path 1 + * @param path2 string representing a file path 2 + * @param path3 string representing a file path 3 + * @return proper os-dependent file path + */ + std::string combine(const std::string& path1, const std::string& path2, const std::string& path3) + { + return combine(combine(path1, path2), path3); + } namespace detail { #ifndef WIN32 /** Helper functor to match path separator on Linux diff --git a/src/interop/util/time.cpp b/src/interop/util/time.cpp index 163696857..ae970661d 100644 --- a/src/interop/util/time.cpp +++ b/src/interop/util/time.cpp @@ -60,7 +60,7 @@ namespace illumina { namespace interop { namespace util */ ::uint64_t csharp_date_time::to_unix(const ::uint64_t val) { - static_assert(sizeof(uint64_t) == 8, "Int64 has the wrong size"); + static_assert(sizeof( ::uint64_t ) == 8, "Int64 has the wrong size"); int64_t ticks = static_cast(val) & INTEROP_UTIL_TICKS_MASK; if (ticks > INTEROP_UTIL_TICKS_THRESHOLD) ticks -= INTEROP_UTIL_TICKS_OFFSET; @@ -69,7 +69,7 @@ namespace illumina { namespace interop { namespace util { ticks += INTEROP_UTIL_TICKS_NEG_OFFSET; } - return static_cast( (ticks - ticks_to_1970()) / ticks_per_second() ); + return static_cast< ::uint64_t >( (ticks - ticks_to_1970()) / ticks_per_second() ); } /** Convert to seconds with fractions * @@ -146,7 +146,7 @@ namespace illumina { namespace interop { namespace util std::istream& operator>>(std::istream& in, csharp_date_time& date_time) { // TODO: read in nice string representation - uint64_t val; + ::uint64_t val; in >> val; date_time = csharp_date_time(val); return in; diff --git a/src/tests/csharp/CMakeLists.txt b/src/tests/csharp/CMakeLists.txt index 4ade01390..626b104bf 100644 --- a/src/tests/csharp/CMakeLists.txt +++ b/src/tests/csharp/CMakeLists.txt @@ -31,6 +31,7 @@ set(TEST_SRCS logic/PlotQScoreHistogram.cs logic/PlotDataByLaneTest.cs logic/PlotDataByCycleTest.cs + metrics/RunMetricsTest.cs metrics/ErrorMetricsTest.cs metrics/CorrectedIntensityMetricsTest.cs metrics/ExtractionMetricsTest.cs @@ -38,6 +39,8 @@ set(TEST_SRCS metrics/IndexMetricsTest.cs metrics/QMetricsTest.cs metrics/TileMetricsTest.cs + run/RunInfoTest.cs + run/RunParametersTest.cs ) set(csharp_files) foreach(SRC ${TEST_SRCS}) @@ -89,4 +92,4 @@ set_target_properties(check_csharp PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DE set(NUNIT_COMMAND_EXE "${NUNIT_COMMAND}" PARENT_SCOPE) -set(CSHARP_PERF_TEST_EXE "${CSHARP_csharp_perftest_BINARY_NAME}" PARENT_SCOPE) \ No newline at end of file +set(CSHARP_PERF_TEST_EXE "${CSHARP_csharp_perftest_BINARY_NAME}" PARENT_SCOPE) diff --git a/src/tests/csharp/logic/ExceptionTest.cs b/src/tests/csharp/logic/ExceptionTest.cs index 98a43141a..a8565d455 100644 --- a/src/tests/csharp/logic/ExceptionTest.cs +++ b/src/tests/csharp/logic/ExceptionTest.cs @@ -4,7 +4,7 @@ using Illumina.InterOp.Run; using Illumina.InterOp.Metrics; using Illumina.InterOp.Plot; -using Illumina.InterOp.Imaging; +using Illumina.InterOp.Table; using Illumina.InterOp.Comm; namespace Illumina.InterOp.Interop.UnitTest @@ -54,6 +54,7 @@ public void TestInvalidFilterOption() { run_metrics metrics = new run_metrics(); filter_options options = new filter_options(tile_naming_method.FourDigit); + metrics.extraction_metric_set().insert(new extraction_metric(1,1101,1, 0L, new ushort_vector(), new float_vector())); candle_stick_plot_data data = new candle_stick_plot_data(); options.cycle(1); c_csharp_plot.plot_by_cycle(metrics, metric_type.Intensity, options, data); @@ -80,7 +81,7 @@ public void TestInvalidColumnType() size_vector_2d offsets = new size_vector_2d(); column_header_vector headers = new column_header_vector(); headers.Add(new column_header("NoColumn", column_data_type.UnknownDataType)); - c_csharp_imaging.populate_column_offsets(offsets, headers); + c_csharp_table.populate_column_offsets(offsets, headers); }*/ } } diff --git a/src/tests/csharp/logic/ImagingTableLogic.cs b/src/tests/csharp/logic/ImagingTableLogic.cs index 9cb1c5b76..f7e0c0b4b 100644 --- a/src/tests/csharp/logic/ImagingTableLogic.cs +++ b/src/tests/csharp/logic/ImagingTableLogic.cs @@ -1,7 +1,7 @@ using System; using NUnit.Framework; using System.IO; -using Illumina.InterOp.Imaging; +using Illumina.InterOp.Table; using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; using Illumina.InterOp.Comm; @@ -33,7 +33,7 @@ public void PopulateFloatBackedTableTest() var run_param = new parameters(); Assert.AreEqual(run_param.version(), 0); - Assert.AreEqual(run.extraction_metric_set().MaxCycle, 1); + Assert.AreEqual(run.extraction_metric_set().max_cycle(), 1); read_info_vector reads = new read_info_vector(); reads.Add(new read_info(1, 1, 26)); @@ -50,12 +50,12 @@ public void PopulateFloatBackedTableTest() run.legacy_channel_update(instrument_type.HiSeq); imaging_column_vector columnVector = new imaging_column_vector(); - c_csharp_imaging.create_imaging_table_columns(run, columnVector); + c_csharp_table.create_imaging_table_columns(run, columnVector); map_id_offset rowOffsets = new map_id_offset(); - c_csharp_imaging.count_table_rows(run, rowOffsets); - uint columnCount = c_csharp_imaging.count_table_columns(columnVector); + c_csharp_table.count_table_rows(run, rowOffsets); + uint columnCount = c_csharp_table.count_table_columns(columnVector); var data = new float[rowOffsets.Count*columnCount]; - c_csharp_imaging.populate_imaging_table_data(run, columnVector, rowOffsets, data, (uint)data.Length); + c_csharp_table.populate_imaging_table_data(run, columnVector, rowOffsets, data, (uint)data.Length); Assert.AreEqual(rowOffsets.Count, 3); Assert.AreEqual(data[0], 7); @@ -79,7 +79,7 @@ public void PopulateTableTest() var run_param = new parameters(); Assert.AreEqual(run_param.version(), 0); - Assert.AreEqual(run.extraction_metric_set().MaxCycle, 1); + Assert.AreEqual(run.extraction_metric_set().max_cycle(), 1); read_info_vector reads = new read_info_vector(); reads.Add(new read_info(1, 1, 26)); @@ -96,7 +96,7 @@ public void PopulateTableTest() run.legacy_channel_update(instrument_type.HiSeq); imaging_table table = new imaging_table(); - c_csharp_imaging.create_imaging_table(run, table); + c_csharp_table.create_imaging_table(run, table); Assert.AreEqual(table.row_count(), 3); Assert.AreEqual(table.at(0, 0), 7); diff --git a/src/tests/csharp/logic/PlotDataByCycleTest.cs b/src/tests/csharp/logic/PlotDataByCycleTest.cs index 26b40ca83..d74a53b72 100644 --- a/src/tests/csharp/logic/PlotDataByCycleTest.cs +++ b/src/tests/csharp/logic/PlotDataByCycleTest.cs @@ -39,7 +39,7 @@ public void IntensityTest() var run_param = new parameters(); Assert.AreEqual(run_param.version(), 0); - Assert.AreEqual(run.extraction_metric_set().MaxCycle, 1); + Assert.AreEqual(run.extraction_metric_set().max_cycle(), 1); filter_options options = new filter_options(tile_naming_method.FourDigit); read_info_vector reads = new read_info_vector(); diff --git a/src/tests/csharp/metrics/CorrectedIntensityMetricsTest.cs b/src/tests/csharp/metrics/CorrectedIntensityMetricsTest.cs index c197d5634..e007c6698 100644 --- a/src/tests/csharp/metrics/CorrectedIntensityMetricsTest.cs +++ b/src/tests/csharp/metrics/CorrectedIntensityMetricsTest.cs @@ -25,7 +25,6 @@ public abstract class AbstractCorrectedIntensityMetricsTest /// protected vector_corrected_intensity_metrics expected_metrics = new vector_corrected_intensity_metrics(); base_corrected_intensity_metrics actual_metric_set = new base_corrected_intensity_metrics(); - vector_corrected_intensity_metrics actual_metrics; byte[] expected_binary_data; /// /// The setup should be overridden by the specific version of the format @@ -45,7 +44,6 @@ protected void SetupBuffers(int[] tmp, short version) for(int i=0;i @@ -58,24 +56,24 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.version(), actual_metric_set.version()); Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); - for(int i=0;i - /// Confirm that GetMetricsByCycle returns the correct result - /// - [Test] - public void TestGetMetricsByCycle() - { - IEnumerable metric = expected_metric_set.GetMetricsByCycle(7,1114, Enumerable.Range(1,3)); - Assert.IsTrue(metric.Any()); - //IEnumerable metric = actual_metric_set.GetMetricsByCycle(7,1114, Enumerable.Range(1,3)); - //Assert.IsTrue(metric.Any()); - } } } diff --git a/src/tests/csharp/metrics/ErrorMetricsTest.cs b/src/tests/csharp/metrics/ErrorMetricsTest.cs index a853fedb4..98a0b1151 100644 --- a/src/tests/csharp/metrics/ErrorMetricsTest.cs +++ b/src/tests/csharp/metrics/ErrorMetricsTest.cs @@ -16,7 +16,6 @@ public class ErrorMetricsTestV3 base_error_metrics expected_metric_set; base_error_metrics actual_metric_set = new base_error_metrics(); vector_error_metrics expected_metrics = new vector_error_metrics(); - vector_error_metrics actual_metrics; byte[] expected_binary_data; /// @@ -41,7 +40,6 @@ protected void SetUp() } expected_metric_set = new base_error_metrics(expected_metrics, Version, header); c_csharp_comm.read_interop_from_buffer(expected_binary_data, (uint)expected_binary_data.Length, actual_metric_set); - actual_metrics = actual_metric_set.metrics(); } /// @@ -54,15 +52,15 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.version(), actual_metric_set.version()); Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); - for(int i=0;i @@ -46,7 +45,6 @@ protected void SetUp() for(int i=0;i @@ -59,16 +57,16 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.version(), actual_metric_set.version()); Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); - for(int i=0;i protected vector_image_metrics expected_metrics = new vector_image_metrics(); base_image_metrics actual_metric_set = new base_image_metrics(); - vector_image_metrics actual_metrics; byte[] expected_binary_data; /// /// The setup should be overridden by the specific version of the format @@ -44,7 +43,6 @@ protected void SetupBuffers(int[] tmp, short version, ushort channelCount) for(int i=0;i @@ -57,16 +55,16 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); Assert.AreEqual(expected_metric_set.channel_count(), actual_metric_set.channel_count()); - for(int i=0;i @@ -52,7 +51,6 @@ protected void SetUp() for(int i=0;i @@ -65,18 +63,18 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.version(), actual_metric_set.version()); Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); - for(int i=0;i protected vector_q_metrics expected_metrics = new vector_q_metrics(); base_q_metrics actual_metric_set = new base_q_metrics(); - vector_q_metrics actual_metrics; byte[] expected_binary_data; /// /// The setup should be overridden by the specific version of the format @@ -45,8 +44,7 @@ protected void SetupBuffers(int[] tmp, short version, q_score_header header) for(int i=0;i /// Confirms that reading the hard coded binary data matches the expected values of the metric @@ -58,24 +56,24 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); Assert.AreEqual(c_csharp_metrics.count_q_metric_bins(expected_metric_set), c_csharp_metrics.count_q_metric_bins(actual_metric_set)); - Assert.AreEqual(expected_metric_set.binCount(), actual_metric_set.binCount()); + Assert.AreEqual(expected_metric_set.bin_count(), actual_metric_set.bin_count()); - for(int i=0;i @@ -102,14 +100,14 @@ public class QMetricsTestV4 : AbstractQMetricsTest [SetUp] protected override void SetUp() { - const uint binCount = 50; + const uint bin_count = 50; QVal[] hist1 = new QVal[]{new QVal(14,21208), new QVal(21,8227), new QVal(26,73051), new QVal(32,2339486)}; QVal[] hist2 = new QVal[]{new QVal(14,22647), new QVal(21,9570), new QVal(26,81839), new QVal(32,2413227)}; QVal[] hist3 = new QVal[]{new QVal(14,18878), new QVal(21,8168), new QVal(26,72634), new QVal(32,2342292)}; - expected_metrics.Add(new q_metric(1, 1104, 1, ToHist(hist1), binCount)); - expected_metrics.Add(new q_metric(1, 1106, 1, ToHist(hist2), binCount)); - expected_metrics.Add(new q_metric(1, 1104, 2, ToHist(hist3), binCount)); + expected_metrics.Add(new q_metric(1, 1104, 1, ToHist(hist1), bin_count)); + expected_metrics.Add(new q_metric(1, 1106, 1, ToHist(hist2), bin_count)); + expected_metrics.Add(new q_metric(1, 1104, 2, ToHist(hist3), bin_count)); int[] tmp = new int[]{ 4,206, @@ -145,14 +143,14 @@ public class QMetricsTestV5 : AbstractQMetricsTest [SetUp] protected override void SetUp() { - const uint binCount = 7; + const uint bin_count = 7; QVal[] hist1 = new QVal[]{new QVal(1,45272), new QVal(3,33369), new QVal(4,1784241)}; QVal[] hist2 = new QVal[]{new QVal(1,45229), new QVal(3,34304), new QVal(4,1792186)}; QVal[] hist3 = new QVal[]{new QVal(1,49152), new QVal(3,37440), new QVal(4,1806479)}; - expected_metrics.Add(new q_metric(1, 1103, 1, ToHist(hist1), binCount)); - expected_metrics.Add(new q_metric(1, 1104, 1, ToHist(hist2), binCount)); - expected_metrics.Add(new q_metric(1, 1108, 1, ToHist(hist3), binCount)); + expected_metrics.Add(new q_metric(1, 1103, 1, ToHist(hist1), bin_count)); + expected_metrics.Add(new q_metric(1, 1104, 1, ToHist(hist2), bin_count)); + expected_metrics.Add(new q_metric(1, 1108, 1, ToHist(hist3), bin_count)); ushort[] lower = new ushort[]{1, 10, 20, 25, 30, 35, 40}; diff --git a/src/tests/csharp/metrics/RunMetricsTest.cs b/src/tests/csharp/metrics/RunMetricsTest.cs new file mode 100644 index 000000000..f04c76bbd --- /dev/null +++ b/src/tests/csharp/metrics/RunMetricsTest.cs @@ -0,0 +1,52 @@ +using System; +using NUnit.Framework; +using System.IO; +using Illumina.InterOp.Run; +using Illumina.InterOp.Metrics; +using Illumina.InterOp.Comm; + +namespace Illumina.InterOp.Interop.UnitTest +{ + /// + /// Test C# Swig run_metrics Mapping + /// + [TestFixture] + public class RunMetricsTests + { + /// + /// Test list_filenames + /// + [Test] + public void TestListErrorMetricFilenames() + { + run_metrics run = new run_metrics(); + + read_info_vector reads = new read_info_vector(); + reads.Add(new read_info(1, 1, 3)); + run.run_info(new info( + "", + "", + 1, + new flowcell_layout(2, 2, 2, 16), + new string_vector(), + new image_dimensions(), + reads + )); + run.set_naming_method(tile_naming_method.FourDigit); + run.legacy_channel_update(instrument_type.HiSeq); + + string_vector filenames = new string_vector(); + run.list_error_metric_filenames(filenames, "RunFolder"); + Assert.AreEqual(filenames.Count, 1); + Assert.AreEqual(filenames[0], Path.Combine("RunFolder", "InterOp", "ErrorMetricsOut.bin")); + + } + [Test] + public void RunInfoPath() + { + Assert.AreEqual(Path.Combine("RunFolder", "RunInfo.xml"), paths.run_info("RunFolder")); + } + } +} + + diff --git a/src/tests/csharp/metrics/TileMetricsTest.cs b/src/tests/csharp/metrics/TileMetricsTest.cs index a5616a80c..bc99e5f05 100644 --- a/src/tests/csharp/metrics/TileMetricsTest.cs +++ b/src/tests/csharp/metrics/TileMetricsTest.cs @@ -16,7 +16,6 @@ public class TileMetricsTestV2 base_tile_metrics expected_metric_set; base_tile_metrics actual_metric_set = new base_tile_metrics(); vector_tile_metrics expected_metrics = new vector_tile_metrics(); - vector_tile_metrics actual_metrics; byte[] expected_binary_data; /// @@ -63,7 +62,6 @@ protected void SetUp() for(int i=0;i @@ -76,21 +74,21 @@ public void TestHardCodedBinaryData() Assert.AreEqual(expected_metric_set.version(), actual_metric_set.version()); Assert.AreEqual(expected_metric_set.size(), actual_metric_set.size()); - for(int i=0;i + /// Confirm that the run info XML parsing works properly in C# + /// + /// + [TestFixture] + public abstract class AbstractRunInfoTest + { + /// + /// The expected run info + /// + protected info expected_run_info; + /// + /// The actual run info + /// + protected info run_info = new info(); + /// + /// The setup should be overridden by the specific configuration file + /// + [SetUp] + protected abstract void SetUp(); + + /// + /// Confirms that reading the hard coded binary data matches the expected values of the metric + /// + [Test] + public void CompareRunInfo() + { + Assert.AreEqual(run_info.name(), expected_run_info.name()); + Assert.AreEqual(run_info.date(), expected_run_info.date()); + Assert.AreEqual(run_info.version(), expected_run_info.version()); + Assert.AreEqual(run_info.flowcell().lane_count(), expected_run_info.flowcell().lane_count()); + Assert.AreEqual(run_info.flowcell().surface_count(), expected_run_info.flowcell().surface_count()); + Assert.AreEqual(run_info.flowcell().swath_count(), expected_run_info.flowcell().swath_count()); + Assert.AreEqual(run_info.flowcell().tile_count(), expected_run_info.flowcell().tile_count()); + Assert.AreEqual(run_info.flowcell().sections_per_lane(), expected_run_info.flowcell().sections_per_lane()); + Assert.AreEqual(run_info.flowcell().lanes_per_section(), expected_run_info.flowcell().lanes_per_section()); + Assert.AreEqual(run_info.flowcell().naming_method(), expected_run_info.flowcell().naming_method()); + Assert.AreEqual(run_info.dimensions_of_image().width(), expected_run_info.dimensions_of_image().width()); + Assert.AreEqual(run_info.dimensions_of_image().height(), expected_run_info.dimensions_of_image().height()); + Assert.AreEqual(run_info.channels().Count, expected_run_info.channels().Count); + + for(int i=0;i<(int)Math.Min(run_info.channels().Count, expected_run_info.channels().Count);++i) + Assert.AreEqual(run_info.channels()[i], expected_run_info.channels()[i]); + Assert.AreEqual(run_info.reads().Count, expected_run_info.reads().Count); + for(int i=0;i + /// Confirm that the run info XML parsing for MiSeq works properly in C# + /// + /// + [TestFixture] + public class RunInfoMiSeqTest : AbstractRunInfoTest + { + /// + /// The setup should be overridden by the specific version of the format + /// + [SetUp] + protected override void SetUp() + { + string xml_file = "\n"+ + "\n"+ + " \n"+ + " 000000000-A12V4\n"+ + " M00903\n"+ + " 120705\n"+ + " \n"+ + " \n"+ + " \n"+ + " \n"+ + " \n"+ + " \n"+ + " \n"+ + ""; + + string_vector Tiles = new string_vector(); + string_vector ImageChannels = new string_vector(); + read_info[] ReadsArr = new read_info[]{ + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + new read_info(1, 1, 251, false), + new read_info(2, 252, 263, true), + new read_info(3, 264, 514, false) + }; + read_info_vector Reads = new read_info_vector(); + for(int i=0;i + /// Confirm that the run info XML parsing works properly in C# + /// + /// + [TestFixture] + public abstract class AbstractRunParametersTest + { + /// + /// The expected run parameters + /// + protected parameters expected_param; + /// + /// The actual run parameters + /// + protected parameters actual_param = new parameters(); + /// + /// The setup should be overridden by the specific configuration file + /// + [SetUp] + protected abstract void SetUp(); + + /// + /// Confirms that reading the hard coded binary data matches the expected values of the metric + /// + [Test] + public void CompareRunInfo() + { + Assert.AreNotEqual(actual_param.instrument_type(), instrument_type.UnknownInstrument); + Assert.AreEqual(actual_param.instrument_type(), expected_param.instrument_type()); + + Assert.AreEqual(actual_param.version(), expected_param.version()); + } + } + /// + /// Confirm that the run info XML parsing for MiSeq works properly in C# + /// + /// + [TestFixture] + public class RunParametersMiSeqTest : AbstractRunParametersTest + { + /// + /// The setup should be overridden by the specific version of the format + /// + [SetUp] + protected override void SetUp() + { + string xml_file = "\n"+ + "\n"+ + " false\n"+ + " 000000000-A12V4\n"+ + " true\n"+ + " \n"+ + " 000000000-A12V4\n"+ + " 15028382\n"+ + " 2012-10-09T00:00:00\n"+ + " \n"+ + " \n"+ + " MS2106444-00PR2\n"+ + " 15023445\n"+ + " 2013-06-18T00:00:00\n"+ + " \n"+ + " \n"+ + " 0001-01-01T00:00:00\n"+ + " \n"+ + " true\n"+ + " \n"+ + " Post-Run Wash\n"+ + " \n"+ + " false\n"+ + " 90.0.1.44\n"+ + " 14\n"+ + " 1\n"+ + " 1\n"+ + " 0\n"+ + " 0\n"+ + " 0\n"+ + " 0\n"+ + " MiSeq Control Software\n"+ + " \n"+ + " 120705_M00903_0009_A000000000-A12V4\n"+ + " M00903\n"+ + " 8\n"+ + " 9.5.10\n"+ + " Not Specified\n"+ + " Not Specified\n"+ + " Not Specified\n"+ + " Not Specified\n"+ + " 90.0.1.44\n"+ + " 1.16.18\n"+ + " MS2106444-00PR2\n"+ + " \n"+ + " Version2\n"+ + " AA1234567-12345\n"+ + " \n"+ + " \n"+ + " \n"+ + " Default\n"+ + " XXXXX\n"+ + " \n"+ + " 0\n"+ + " \n"+ + " \n"+ + " Metagenomics\n"+ + " \n"+ + " false\n"+ + " \n"+ + " \n"+ + " \n"+ + " \n"+ + " \n"+ + " D:\\Illumina\\MiSeqTemp\\120705_M00903_0009_A000000000-A12V4\n"+ + " D:\\Illumina\\MiSeqAnalysis\\120705_M00903_0009_A000000000-A12V4\n"+ + " 120705\n"+ + " PostRun\n"+ + " D:\\Illumina\\MiSeq Control Software\\CustomRecipe\n"+ + " C:\\Illumina\\MiSeq Control Software\\Recipe\n"+ + " SampleSheet_Miseq\n"+ + " C:\\Illumina\\MiSeq Control Software\\SampleSheets\n"+ + " D:\\Illumina\\MiSeq Control Software\\Manifests\n"+ + " Z:\\120705_M00903_0009_A000000000-A12V4\n"+ + " A\n"+ + " AutoFocus\n"+ + " Both\n"+ + " true\n"+ + " true\n"+ + " \n"+ + " 398407\n"+ + " true\n"+ + " \n"+ + ""; + + + uint Version = 0; + instrument_type ApplicationName_and_SupportMultipleSurfacesInUI=instrument_type.MiSeq; + expected_param = new parameters(Version, + ApplicationName_and_SupportMultipleSurfacesInUI); + actual_param.parse(xml_file); + + } + } +} diff --git a/src/tests/interop/CMakeLists.txt b/src/tests/interop/CMakeLists.txt index 208bb589e..4cdfc622d 100644 --- a/src/tests/interop/CMakeLists.txt +++ b/src/tests/interop/CMakeLists.txt @@ -5,8 +5,11 @@ include(${GMOCK_USE_FILE}) find_package(Threads) set(SRCS + unit_tests.cpp run/info_test.cpp run/parameters_test.cpp + util/option_parser_test.cpp + util/stat_test.cpp metrics/corrected_intensity_metrics_test.cpp metrics/error_metrics_test.cpp metrics/extraction_metrics_test.cpp @@ -16,18 +19,24 @@ set(SRCS metrics/tile_metrics_test.cpp metrics/q_by_lane_metric_test.cpp metrics/q_collapsed_metrics_test.cpp - logic/summary_metrics_test.cpp metrics/base_metric_tests.cpp + metrics/run_metric_test.cpp + metrics/metric_streams_test.cpp + logic/plot_candle_stick_test.cpp + logic/summary_metrics_test.cpp logic/enum_parsing_test.cpp + logic/channel_test.cpp logic/metric_type_ext_test.cpp logic/plot_logic_test.cpp - util/option_parser_test.cpp logic/imaging_table_logic_test.cpp logic/imaging_table_regression_test.cpp - metrics/metric_streams_test.cpp - util/stat_test.cpp - unit_tests.cpp - metrics/run_metric_test.cpp) + logic/plot_bar_test.cpp + logic/plot_heatmap_test.cpp + logic/plot_flowcell_test.cpp + logic/index_summary_test.cpp + metrics/coverage_test.cpp + metrics/metric_stream_error_test.cpp + ) set(HEADERS metrics/inc/corrected_intensity_metrics_test.h @@ -46,7 +55,12 @@ set(HEADERS inc/regression_test_data.h inc/proxy_parameter_generator.h inc/abstract_regression_test_generator.h - metrics/inc/metric_format_fixtures.h) + metrics/inc/metric_format_fixtures.h + logic/inc/metric_filter_iterator.h + logic/inc/empty_plot_test_generator.h + logic/inc/plot_regression_test_generator.h + metrics/inc/format_registry.h + ) add_executable(interop_gtests ${SRCS} ${HEADERS}) diff --git a/src/tests/interop/inc/abstract_regression_test_generator.h b/src/tests/interop/inc/abstract_regression_test_generator.h index e2e8867f4..045f56efe 100644 --- a/src/tests/interop/inc/abstract_regression_test_generator.h +++ b/src/tests/interop/inc/abstract_regression_test_generator.h @@ -21,7 +21,8 @@ namespace illumina{ namespace interop { namespace unittest class abstract_regression_test_generator : public abstract_generator< Model > { public: - typedef typename abstract_generator< Model >::base_type base_type; + /** Define the abstract base type */ + typedef typename abstract_generator< Model >::parent_type base_t; public: /** Constructor * @@ -45,58 +46,78 @@ namespace illumina{ namespace interop { namespace unittest abstract_generator< Model >(test_modifier), m_run_folder(run_folder), m_test_dir(test_dir) { } - /** Clone the concret implementation + /** Clone the concrete implementation * * @param name run folder * @return copy of this object */ - base_type operator()(const std::string& name)const + base_t operator()(const std::string& name)const { m_run_folder=name; - return clone(); + return clone(name); } + /** Clone the concrete implementation + * + * @param name run folder + * @return copy of this object + */ + virtual base_t clone(const std::string& name)const=0; /** Generate the expected and actual metric sets * * @param expected expected metric set * @param actual actual metric set * @return true if the results should be tested (false if rebaselining) */ - bool generate(Model& expected, Model& actual)const + ::testing::AssertionResult generate(Model& expected, Model& actual, bool* skip_test)const { const regression_test_data& data = regression_test_data::instance(); const std::string baseline_file = baseline(); + ::testing::Message msg; if(!data.rebaseline()) { - if(io::is_file_readable(baseline_file)) + const bool expected_found = read_expected(baseline_file, expected); + const bool actual_generated = generate_actual(m_run_folder, actual); + if(expected_found == actual_generated) { - read_expected(baseline_file, expected); - generate_actual(m_run_folder, actual); - return true; + *skip_test = !expected_found; + return ::testing::AssertionSuccess(); } - else EXPECT_TRUE(false) << "Failed to find baseline: " << baseline_file; + if(!expected_found) + return ::testing::AssertionFailure() << "Failed to find baseline: " << baseline_file; + return ::testing::AssertionFailure() << "Failed to generate data for baseline: " << baseline_file; } else { - std::cout << "[ ] Rebaseline: " << *this << std::endl; try { - generate_actual(m_run_folder, actual); - EXPECT_TRUE(write_actual(baseline_file, actual)) << "Failed to write baseline: " << baseline_file; + if(generate_actual(m_run_folder, actual)) + { + msg << "[ ] Rebaseline: " << *this; + if(!write_actual(baseline_file, actual)) + return ::testing::AssertionFailure() << "Failed to write baseline: " << baseline_file; + } + else + { + msg << "[ ] Skipped: " << *this; + } + *skip_test = true; } catch(const std::exception& ex) { - EXPECT_TRUE(false)<< "Failed to generate baseline: " << baseline_file << " " << ex.what(); + std::cerr << "Failed to generate baseline: " << baseline_file << " " << ex.what() << std::endl; + std::exit(1); + //return ::testing::AssertionFailure() << "Failed to generate baseline: " << baseline_file << " " << ex.what(); } } - return false; + return ::testing::AssertionFailure(msg); } /** Get the full path of the baseline output file * * @return full path */ - std::string baseline()const + virtual std::string baseline()const { - return io::combine(io::combine(regression_test_data::instance().baseline(), m_test_dir), io::basename(m_run_folder)); + return io::combine(io::combine(io::combine(regression_test_data::instance().baseline(), io::basename(m_run_folder)), m_test_dir), "baseline"); } protected: @@ -104,14 +125,16 @@ namespace illumina{ namespace interop { namespace unittest * * @param baseline_file baseline file * @param expected expected model data + * @return true if the file was found, and the read completed without failure */ - virtual void read_expected(const std::string& baseline_file, Model& expected)const=0; + virtual bool read_expected(const std::string& baseline_file, Model& expected)const=0; /** Read the actual data from the run folder * * @param run_folder run folder * @param actual actual model data + * @return true if data was generated */ - virtual void generate_actual(const std::string& run_folder, Model& actual)const=0; + virtual bool generate_actual(const std::string& run_folder, Model& actual)const=0; /** Write the actual data to the run folder * * @param baseline_file baseline file @@ -122,7 +145,7 @@ namespace illumina{ namespace interop { namespace unittest * * @return pointer to copy */ - virtual base_type clone()const=0; + virtual base_t clone()const=0; protected: /** Run Folder name */ diff --git a/src/tests/interop/inc/generic_fixture.h b/src/tests/interop/inc/generic_fixture.h index 3c20a0dfa..5fe6ccedf 100644 --- a/src/tests/interop/inc/generic_fixture.h +++ b/src/tests/interop/inc/generic_fixture.h @@ -9,6 +9,7 @@ #include "interop/util/assert.h" #include "interop/util/unique_ptr.h" +#include "interop/util/math.h" namespace illumina{ namespace interop { namespace unittest { @@ -21,28 +22,38 @@ namespace illumina{ namespace interop { namespace unittest { template class abstract_generator { - public: + /** Pointer to abtract generator */ typedef abstract_generator* base_type; + public: + /** Pointer to abtract generator */ typedef generator_ptr parent_type; /** Constructor * * @param test_modifier flag that modifies the test */ abstract_generator(const int test_modifier=0) : m_test_modifier(test_modifier){} + /** Destructor */ + virtual ~abstract_generator(){} /** Generate the expected and actual metric sets * * @param expected expected object * @param actual actual object */ - virtual bool generate(T& expected, T& actual)const=0; + virtual ::testing::AssertionResult generate(T& expected, T& actual, bool* skip_test)const=0; + /** Advance to the next type + * + * @return true when the generator has finished, and the next parameter can be obtained + */ + virtual bool advance() + { + return true; + } /** Create a copy of this object * * @return pointer to copy */ - virtual base_type clone()const=0; - /** Destructor */ - virtual ~abstract_generator(){} + virtual parent_type clone()const=0; /** Flag that modifies the test * * @return flag that modifies the test @@ -69,6 +80,92 @@ namespace illumina{ namespace interop { namespace unittest { int m_test_modifier; }; + /** Generate the plot from an empty metric set + * + * The expected plot is empty + */ + template + class standard_parameter_generator : public abstract_generator + { + typedef typename Fixture::parameter_type parameter_type; + typedef typename abstract_generator::parent_type base_type; + public: + /** Constructor + * + * @param parameter parameter value + */ + standard_parameter_generator(const parameter_type parameter) : m_fixture(parameter) + { + } + /** Clone the concrete implementation TODO: Remove + * + * @param name run folder + * @return copy of this object + */ + base_type operator()(const parameter_type& parameter)const + { + return new standard_parameter_generator(parameter); + } + + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(T& expected, T &actual, bool*) const + { + return m_fixture.generate(expected, actual); + } + + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + base_type clone() const + { + return new standard_parameter_generator(*this); + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + m_fixture.write(out); + } + + protected: + /** Fixture */ + Fixture m_fixture; + }; + /** Generate the plot from an empty metric set + * + * The expected plot is empty + */ + template + class iterator_parameter_generator : public standard_parameter_generator + { + typedef typename Fixture::parameter_type parameter_type; + public: + /** Constructor + * + * @param parameter parameter value + */ + iterator_parameter_generator(const parameter_type parameter) : + standard_parameter_generator(parameter) + { + } + /** Advance to the next type + * + * @return true when the generator has finished, and the next parameter can be obtained + */ + bool advance() + { + return standard_parameter_generator::m_fixture.advance(); + } + }; + /** Smart pointer wrapper * * Performs deep copies @@ -86,7 +183,15 @@ namespace illumina{ namespace interop { namespace unittest { * * @param other source object to copy */ - generator_ptr(const generator_ptr& other) : m_ptr(other.m_ptr==0?0:other.m_ptr->clone()){} + generator_ptr(const generator_ptr& other) : m_ptr(0) + { + if(other.m_ptr != 0) + { + generator_ptr tmp = other.m_ptr->clone(); + m_ptr = tmp.m_ptr; + tmp.m_ptr=0; + } + } /** Copy operator * * @param other source object to copy @@ -96,7 +201,24 @@ namespace illumina{ namespace interop { namespace unittest { { delete m_ptr; INTEROP_ASSERT(other.m_ptr != 0); - if(other.m_ptr != 0) m_ptr= other.m_ptr->clone(); + + if(other.m_ptr != 0) + { + generator_ptr tmp = other.m_ptr->clone(); + m_ptr = tmp.m_ptr; + tmp.m_ptr=0; + } + return *this; + } + /** Copy operator + * + * @param other source object to copy + * @return this + */ + generator_ptr& operator=(abstract_generator* ptr) + { + delete m_ptr; + if(ptr != 0) m_ptr = ptr; return *this; } /** Destructor */ @@ -138,6 +260,24 @@ namespace illumina{ namespace interop { namespace unittest { INTEROP_ASSERT(m_ptr != 0); return m_ptr; } + /** Test if current pointer equals another + * + * @param other other pointer + * @return true if they point to same address + */ + bool operator==(const abstract_generator* other) + { + return m_ptr == other; + } + /** Test if current pointer does not equal another + * + * @param other + * @return true if they do not point to same address + */ + bool operator!=(const abstract_generator* other) + { + return m_ptr != other; + } /** Write name of generator to output stream * * @param out output stream @@ -174,8 +314,9 @@ namespace illumina{ namespace interop { namespace unittest { template struct generic_test_fixture : public ::testing::TestWithParam< generator_ptr > { - private: - typedef ::testing::TestWithParam< generator_ptr > parent_type; + public: + /** Get the type of the parent */ + typedef ::testing::TestWithParam< generator_ptr > parent_type; public: //typedef abstract_generator* generator_type; typedef generator_ptr generator_type; @@ -184,21 +325,80 @@ namespace illumina{ namespace interop { namespace unittest { /** Constructor */ - generic_test_fixture() + generic_test_fixture() : skip_test(false), + fixture_test_result(parent_type::GetParam()->generate(expected, actual, &skip_test)) { - test = parent_type::GetParam()->generate(expected, actual); + if(skip_test && !fixture_test_result) + std::cout << fixture_test_result.message() << std::endl; test_modifier = parent_type::GetParam()->test_modifier(); } /** Expected object to test */ T expected; /** Actual object to test */ T actual; - /** Run test */ - bool test; + /** Skip running the test */ + bool skip_test; + ::testing::Message msg; /** Flag for type of test*/ int test_modifier; + /** Error trying to generate the fixture */ + const ::testing::AssertionResult fixture_test_result; }; + /** Check if two floats are nearly the same. If both are NaN, then this check succeeds + * + * @todo Use this everywhere + * + * @param expected expected float + * @param actual actual float + * @param tol tolerance + * @return true if both numbers hold the same value, or their difference is less than tolerance + */ + inline ::testing::AssertionResult AreFloatsNear(const float expected, const float actual, const float tol) + { + if(std::isnan(expected) && std::isnan(actual)) return ::testing::AssertionSuccess(); + if(std::isnan(expected) || std::isnan(actual)) + return ::testing::AssertionFailure() << "Abs(" << expected << " - " << actual << ") >= " << tol; + if(std::abs(expected-actual) < tol) return ::testing::AssertionSuccess(); + return ::testing::AssertionFailure() << "Abs(" << expected << " - " << actual << ") >= " << tol; + } + + /** Check if two float arrays are nearly the same. If both are NaN, then this check succeeds + * + * @todo Use this everywhere + * + * @param expected expected vector of floats + * @param actual actual vector of floats + * @param tol tolerance + * @return true if both numbers hold the same value, or their difference is less than tolerance + */ + inline ::testing::AssertionResult AreValuesNear(const std::vector& expected, + const std::vector& actual, + const float tol) + { + ::testing::Message msg; + bool test_failed = false; + if( expected.size() != actual.size() ) + { + return ::testing::AssertionFailure() << "Expected size: " << expected.size() << " != actual: " << actual.size(); + } + for(size_t i=0;i= tol) + { + if(test_failed) msg << " | "; + msg << "Value("<< i << ") Expected: " << expected[i] << " == Actual: " << actual[i]; + test_failed=true; + } + } + if(test_failed) return ::testing::AssertionFailure(msg << " Tol: " << tol); + return ::testing::AssertionSuccess(); + } + }}} +#define INTEROP_ASSERT_NEAR(EXPECTED, ACTUAL, TOL) ASSERT_TRUE(AreFloatsNear(EXPECTED, ACTUAL, TOL)) +#define INTEROP_EXPECT_NEAR(EXPECTED, ACTUAL, TOL) EXPECT_TRUE(AreFloatsNear(EXPECTED, ACTUAL, TOL)) +#define INTEROP_EXPECT_ARRAY_NEAR(EXPECTED, ACTUAL, TOL) EXPECT_TRUE(AreValuesNear(EXPECTED, ACTUAL, TOL)) diff --git a/src/tests/interop/inc/proxy_parameter_generator.h b/src/tests/interop/inc/proxy_parameter_generator.h index 93eecb8f5..3a9322174 100644 --- a/src/tests/interop/inc/proxy_parameter_generator.h +++ b/src/tests/interop/inc/proxy_parameter_generator.h @@ -9,9 +9,13 @@ */ #pragma once #include +#include "interop/util/type_traits.h" +#include "interop/util/static_assert.h" -namespace illumina{ namespace interop { namespace unittest { +namespace illumina{ namespace interop { namespace unittest +{ + /** Data structure that holds the parameters * * This holds a reference to the input vector, so ensure it is not on the stack! @@ -43,7 +47,6 @@ namespace illumina{ namespace interop { namespace unittest { private: const std::vector& m_vec; - mutable std::vector< typename T::parent_type > m_proxy_data; T& m_object; }; /** Iterator over persistent vector of arguments @@ -53,7 +56,7 @@ namespace illumina{ namespace interop { namespace unittest { template class proxy_argument_iterator : public ::testing::internal::ParamIteratorInterface< typename T::parent_type > { - typedef typename std::vector< typename T::parent_type >::const_iterator const_iterator; + typedef typename std::vector< Proxy >::const_iterator const_iterator; public: /** Constructor * @@ -61,12 +64,19 @@ namespace illumina{ namespace interop { namespace unittest { * @param base generator base * @param it iterator to std::vector */ - proxy_argument_iterator(T& obj, const proxy_argument_generator& base, const_iterator it) : - m_base(base), m_begin(it), m_current(it), m_object(obj) + proxy_argument_iterator(T& obj, const proxy_argument_generator& base, const_iterator it, const_iterator it_end) : + m_base(base), m_begin(it), m_current(it), m_end(it_end), m_object(obj) { + static_assert(!is_pointer::value, "This class does not free memory and should not take a pointer"); + if(m_current < m_end) + { + m_current_val=m_object(*m_current); + } } /** Destructor */ - virtual ~proxy_argument_iterator() {} + virtual ~proxy_argument_iterator() + { + } /** A pointer to the base generator instance. * Used only for the purposes of iterator comparison * to make sure that two iterators belong to the same generator. @@ -86,7 +96,14 @@ namespace illumina{ namespace interop { namespace unittest { */ virtual void Advance() { - ++m_current; + if(m_current_val == 0 || m_current_val->advance()) + { + ++m_current; + if(m_current < m_end) + { + m_current_val=m_object(*m_current); + } + } } /** Clones the iterator object. Used for implementing copy semantics * of ParamIterator. @@ -106,7 +123,6 @@ namespace illumina{ namespace interop { namespace unittest { */ virtual const typename T::parent_type* Current() const { - m_current_val=*m_current; return &m_current_val; } /** Determines whether the given iterator and other point to the same @@ -121,33 +137,34 @@ namespace illumina{ namespace interop { namespace unittest { return &m_base == other.BaseGenerator() && m_current == dynamic_cast*>(&other)->m_current; } + private: proxy_argument_iterator(const proxy_argument_iterator& other) : m_base(other.m_base), m_begin(other.m_begin), - m_current(other.m_current), m_object(other.m_object) {} + m_current(other.m_current), m_end(other.m_end), m_object(other.m_object) + { + if(m_current < m_end) m_current_val=m_object(*m_current); + } + private: const proxy_argument_generator& m_base; const const_iterator m_begin; const_iterator m_current; - mutable typename T::parent_type m_current_val; + const_iterator m_end; T& m_object; + mutable typename T::parent_type m_current_val; }; template ::testing::internal::ParamIteratorInterface< typename T::parent_type >* proxy_argument_generator::Begin() const { - if(m_proxy_data.empty()) - { - m_proxy_data.resize(m_vec.size()); - for(size_t i=0;i(m_object, *this, m_proxy_data.begin()); + return new proxy_argument_iterator(m_object, *this, m_vec.begin(), m_vec.end()); } template ::testing::internal::ParamIteratorInterface< typename T::parent_type >* proxy_argument_generator::End() const { - return new proxy_argument_iterator(m_object, *this, m_proxy_data.end()); + return new proxy_argument_iterator(m_object, *this, m_vec.end(), m_vec.end()); } /** Generate parameters from a persistent vector (cannot be on the stack!) diff --git a/src/tests/interop/inc/regression_test_data.h b/src/tests/interop/inc/regression_test_data.h index ae8e837d0..7967a2c5f 100644 --- a/src/tests/interop/inc/regression_test_data.h +++ b/src/tests/interop/inc/regression_test_data.h @@ -105,14 +105,19 @@ namespace illumina{ namespace interop { namespace unittest */ void finalize() { + if(!m_rebaseline) return; io::mkdir(m_baseline); - for(std::vector ::const_iterator it = m_sub_dirs.begin();it != m_sub_dirs.end();++it) + for(std::vector ::const_iterator fit = m_files.begin();fit != m_files.end();++fit) { - io::mkdir(io::combine(m_baseline, *it)); + const std::string run_folder = io::combine(m_baseline, io::basename(*fit)); + io::mkdir(run_folder); + for (std::vector::const_iterator sit = m_sub_dirs.begin(); sit != m_sub_dirs.end(); ++sit) + { + io::mkdir(io::combine(run_folder, *sit)); + } } } - private: std::string m_baseline; std::vector m_files; diff --git a/src/tests/interop/logic/channel_test.cpp b/src/tests/interop/logic/channel_test.cpp new file mode 100644 index 000000000..bf81a3806 --- /dev/null +++ b/src/tests/interop/logic/channel_test.cpp @@ -0,0 +1,379 @@ +/** Unit tests for channel logic + * + * @file + * @date 9/10/2016 + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include "interop/logic/utils/channel.h" +#include "interop/logic/utils/metric_type_ext.h" + + +using namespace illumina::interop; + +/** Convert an array to a vector + * + * Determines the length of the stack array automatically. + * + * @param vals array pointer + * @return vector of values + */ +template +inline std::vector to_vector(const T (&vals)[N]) +{ + return std::vector(vals, vals + N); +} + +TEST(channel_test, normalize) +{ + EXPECT_EQ(logic::utils::normalize("A"), "a"); + EXPECT_EQ(logic::utils::normalize("a"), "a"); + EXPECT_EQ(logic::utils::normalize("C"), "c"); + EXPECT_EQ(logic::utils::normalize("c"), "c"); + EXPECT_EQ(logic::utils::normalize("G"), "g"); + EXPECT_EQ(logic::utils::normalize("g"), "g"); + EXPECT_EQ(logic::utils::normalize("T"), "t"); + EXPECT_EQ(logic::utils::normalize("t"), "t"); + EXPECT_EQ(logic::utils::normalize("1"), "1"); + EXPECT_EQ(logic::utils::normalize("2"), "2"); + EXPECT_EQ(logic::utils::normalize(" "), " "); + EXPECT_EQ(logic::utils::normalize(""), ""); + EXPECT_EQ(logic::utils::normalize("RED"), "red"); + EXPECT_EQ(logic::utils::normalize("gReeN"), "green"); + EXPECT_NE(logic::utils::normalize("A"), "c"); +} + +//Standard test, 4 channel system in different order +TEST(channel_test, expected2actual_4channel_valid) +{ + const std::string channel_array[] = {"C", "G", "T", "A"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + logic::utils::expected2actual(channels, map); + + EXPECT_EQ((size_t) map.size(), (size_t) 4); + EXPECT_EQ(map[0], (size_t) 3); + EXPECT_EQ(map[1], (size_t )0); + EXPECT_EQ(map[2], (size_t )1); + EXPECT_EQ(map[3], (size_t )2); +} + +//Standard test, 2 channel system in different order +TEST(channel_test, expected2actual_2channel_valid) +{ + const std::string channel_array[] = {"Red", "Green"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + logic::utils::expected2actual(channels, map); + + EXPECT_EQ((size_t) map.size(), (size_t) 2); + EXPECT_EQ(map[0], (size_t )0); + EXPECT_EQ(map[1], (size_t )1); +} + + +//Test 4 - check full normalization of all letters in channel name +TEST(channel_test, expected2actual_valid_4) +{ + const std::string channel_array[] = {"rEd", "GReEn"}; + const std::vector channels(to_vector(channel_array)); + + std::vector map; + + logic::utils::expected2actual(channels, map); + + EXPECT_EQ((size_t) map.size(), (size_t) 2); + EXPECT_EQ(map[0], (size_t )0); + EXPECT_EQ(map[1], (size_t )1); +} + +//INVALID - bogus channel names +TEST(channel_test, expected2actual_invalid_channel_1) +{ + const std::string channel_array[] = {"Boy", "Girl"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - valid channel names, but invalid channel count +TEST(channel_test, expected2actual_invalid_channel_2) +{ + const std::string channel_array[] = {"A", "C", "G"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - space string instead of valid channel name +TEST(channel_test, expected2actual_invalid_channel_3) +{ + const std::string channel_array[] = {"A", "C", "G", " "}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - empty string for channel +TEST(channel_test, expected2actual_invalid_channel_4) +{ + const std::string channel_array[] = {"A", "C", "G", ""}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - mixing ACGT system with 1/2 system +TEST(channel_test, expected2actual_invalid_channel_5) +{ + const std::string channel_array[] = {"A", "C", "G", "1"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - invalid channel name that starts with valid channel name (T -> TT) +TEST(channel_test, expected2actual_invalid_channel_6) +{ + const std::string channel_array[] = {"A", "C", "G", "TT"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - valid channel names but these two cannot exist by themselves +TEST(channel_test, expected2actual_invalid_channel_7) +{ + const std::string channel_array[] = {"A", "t"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - duplicate channel name with original duplicate not quite being duplicate +TEST(channel_test, expected2actual_invalid_channel_8) +{ + const std::string channel_array[] = {"red", "green", "Red"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - no channels +TEST(channel_test, expected2actual_invalid_channel_9) +{ + const std::vector channels; + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - 1 channel +TEST(channel_test, expected2actual_invalid_channel_10) +{ + const std::string channel_array[] = {"1"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//INVALID - 1 valid channel with a duplicate making it invalid +TEST(channel_test, expected2actual_invalid_channel_11) +{ + const std::string channel_array[] = {"1", "1"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::expected2actual(channels, map), model::invalid_channel_exception); +} + +//Standard test, 4 channel system in different order +TEST(channel_test, actual2expected_4channel_valid) +{ + const std::string channel_array[] = {"C", "G", "T", "A"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + logic::utils::actual2expected(channels, map); + + EXPECT_EQ((size_t) map.size(), (size_t) 4); + EXPECT_EQ(map[0], (size_t) 1); + EXPECT_EQ(map[1], (size_t )2); + EXPECT_EQ(map[2], (size_t )3); + EXPECT_EQ(map[3], (size_t )0); +} + +//Standard test, 2 channel system in different order +TEST(channel_test, actual2expected_2channel_valid) +{ + const std::string channel_array[] = {"red", "green"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + logic::utils::actual2expected(channels, map); + + EXPECT_EQ((size_t) map.size(), (size_t) 2); + EXPECT_EQ(map[0], (size_t )0); + EXPECT_EQ(map[1], (size_t )1); +} + + + +//Test 4 - check full normalization of all letters in channel name +TEST(channel_test, actual2expected_valid_4) +{ + const std::string channel_array[] = {"rEd", "GReEn"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + logic::utils::actual2expected(channels, map); + + EXPECT_EQ((size_t) map.size(), (size_t) 2); + EXPECT_EQ(map[0], (size_t )0); + EXPECT_EQ(map[1], (size_t )1); +} + +//INVALID - bogus channel names +TEST(channel_test, actual2expected_invalid_channel_1) +{ + const std::string channel_array[] = {"Boy", "Girl"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - valid channel names, but invalid channel count +TEST(channel_test, actual2expected_invalid_channel_2) +{ + const std::string channel_array[] = {"A", "C", "G"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - space string instead of valid channel name +TEST(channel_test, actual2expected_invalid_channel_3) +{ + const std::string channel_array[] = {"A", "C", "G", " "}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - empty string for channel +TEST(channel_test, actual2expected_invalid_channel_4) +{ + const std::string channel_array[] = {"A", "C", "G", ""}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - mixing ACGT system with 1/2 system +TEST(channel_test, actual2expected_invalid_channel_5) +{ + const std::string channel_array[] = {"A", "C", "G", "1"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - invalid channel name that starts with valid channel name (T -> TT) +TEST(channel_test, actual2expected_invalid_channel_6) +{ + const std::string channel_array[] = {"A", "C", "G", "TT"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - valid channel names but these two cannot exist by themselves +TEST(channel_test, actual2expected_invalid_channel_7) +{ + const std::string channel_array[] = {"A", "t"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - duplicate channel name with original duplicate not quite being duplicate +TEST(channel_test, actual2expected_invalid_channel_8) +{ + const std::string channel_array[] = {"rEd", "GreEn", "Red"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - no channels +TEST(channel_test, actual2expected_invalid_channel_9) +{ + const std::vector channels; + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - 1 channel +TEST(channel_test, actual2expected_invalid_channel_10) +{ + const std::string channel_array[] = {"1"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//INVALID - 1 valid channel with a duplicate making it invalid +TEST(channel_test, actual2expected_invalid_channel_11) +{ + const std::string channel_array[] = {"1", "1"}; + const std::vector channels(to_vector(channel_array)); + std::vector map; + + EXPECT_THROW(logic::utils::actual2expected(channels, map), model::invalid_channel_exception); +} + +//Test that getting different channels, including edge case channel names, gives the right size for channel array +TEST(channel_test, update_channel_from_instrument) +{ + std::vector channels; + channels = logic::utils::update_channel_from_instrument_type(constants::NextSeq); + EXPECT_EQ(channels.size(), (size_t) 2); + + std::vector channels_2; + channels_2 = logic::utils::update_channel_from_instrument_type(constants::UnknownInstrument); + EXPECT_EQ(channels_2.size(), (size_t) 0); + + std::vector channels_3; + channels_3 = logic::utils::update_channel_from_instrument_type(constants::InstrumentCount); + EXPECT_EQ(channels_3.size(), (size_t) 0); + + std::vector channels_4; + channels_4 = logic::utils::update_channel_from_instrument_type(constants::HiSeq); + EXPECT_EQ(channels_4.size(), (size_t) 4); + + std::vector channels_5; + channels_5 = logic::utils::update_channel_from_instrument_type(constants::HiScan); + EXPECT_EQ(channels_5.size(), (size_t) 4); +} diff --git a/src/tests/interop/logic/enum_parsing_test.cpp b/src/tests/interop/logic/enum_parsing_test.cpp index 50086d81a..7696a0a20 100644 --- a/src/tests/interop/logic/enum_parsing_test.cpp +++ b/src/tests/interop/logic/enum_parsing_test.cpp @@ -27,13 +27,36 @@ TEST(enum_parsing_test, list_enums_intensity) EXPECT_EQ(types[0], constants::Intensity); } +/* Tests to ensure that each metric group is accurately represented by at least 1 member of the group */ TEST(enum_parsing_test, metric_type_to_string) { EXPECT_EQ(constants::to_string(constants::Intensity), std::string("Intensity")); } -TEST(enum_parsing_test, metric_type_to_group) +TEST(enum_parsing_test, metric_type_to_group_Intensity) { EXPECT_EQ(logic::utils::to_group(constants::Intensity), constants::Extraction); } +TEST(enum_parsing_test, metric_type_to_group_Q20Percent) +{ + EXPECT_EQ(logic::utils::to_group(constants::Q20Percent), constants::Q); +} + +TEST(enum_parsing_test, metric_type_to_group_PercentBase) +{ + EXPECT_EQ(logic::utils::to_group(constants::BasePercent), constants::CorrectedInt); +} + +TEST(enum_parsing_test, metric_type_to_group_Clusters) +{ + EXPECT_EQ(logic::utils::to_group(constants::Clusters), constants::Tile); +} + +TEST(enum_parsing_test, metric_type_to_group_ErrorRate) +{ + EXPECT_EQ(logic::utils::to_group(constants::ErrorRate), constants::Error); +} + + + diff --git a/src/tests/interop/logic/imaging_table_logic_test.cpp b/src/tests/interop/logic/imaging_table_logic_test.cpp index 22b578ed6..56a5fe2e9 100644 --- a/src/tests/interop/logic/imaging_table_logic_test.cpp +++ b/src/tests/interop/logic/imaging_table_logic_test.cpp @@ -10,26 +10,12 @@ #include #include "interop/util/length_of.h" #include "interop/logic/table/create_imaging_table.h" -#include "src/tests/interop/metrics/inc/error_metrics_test.h" #include "interop/io/table/imaging_table_csv.h" +#include "src/tests/interop/metrics/inc/error_metrics_test.h" using namespace illumina::interop; using namespace illumina::interop::unittest; - -/** Convert an array to a vector - * - * Determines the length of the stack array automatically. - * - * @param vals array pointer - * @return vector of values - */ -template -inline std::vector to_vector(const T (&vals)[N]) -{ - return std::vector(vals, vals + N); -} - namespace illumina{ namespace interop {namespace model {namespace table { /** Compare whether two imaging columns are equal @@ -70,7 +56,7 @@ void simulate_read_error_metrics(model::metrics::run_metrics& metrics) metrics.legacy_channel_update(constants::HiSeq); metrics.set_naming_method(constants::FourDigit); - unittest::error_metric_v3::create_metric_set(metrics.get_set()); + unittest::error_metric_v3::create_expected(metrics.get()); } /** * @class illumina::interop::model::table::imaging_table @@ -101,7 +87,12 @@ TEST(imaging_table, base_header_test) const std::string expected_subcolumns[] = { "A", "C", "G", "T" }; - EXPECT_THAT(to_vector(expected_subcolumns), ::testing::ElementsAreArray(columns[0].subcolumns())); + EXPECT_THAT(util::to_vector(expected_subcolumns), ::testing::ElementsAreArray(columns[0].subcolumns())); +} + +TEST(imaging_table, max_digits) +{ + EXPECT_EQ(logic::table::max_digits(), 3u); } TEST(imaging_table, test_write_column_name) @@ -189,7 +180,7 @@ TEST(imaging_table, create_imaging_table_columns_error_metrics) model::table::imaging_column(model::table::TileNumberColumn, 8) }; - EXPECT_THAT(to_vector(expected_columns), ::testing::ElementsAreArray(columns)); + EXPECT_THAT(util::to_vector(expected_columns), ::testing::ElementsAreArray(columns)); } TEST(imaging_table, create_imaging_table_error_metrics) { @@ -197,7 +188,7 @@ TEST(imaging_table, create_imaging_table_error_metrics) simulate_read_error_metrics(metrics); std::vector columns; - std::map row_offsets; + logic::table::row_offset_map_t row_offsets; logic::table::create_imaging_table_columns(metrics, columns); const size_t column_count = logic::table::count_table_columns(columns); logic::table::count_table_rows(metrics, row_offsets); diff --git a/src/tests/interop/logic/imaging_table_regression_test.cpp b/src/tests/interop/logic/imaging_table_regression_test.cpp index a4bd9ca28..4ed89af98 100644 --- a/src/tests/interop/logic/imaging_table_regression_test.cpp +++ b/src/tests/interop/logic/imaging_table_regression_test.cpp @@ -22,6 +22,12 @@ namespace illumina{ namespace interop {namespace model {namespace table bool operator==(const imaging_column& lhs, const imaging_column& rhs); }}}} +/** Simulate reading error metrics + * + * @param metrics run metrics + */ +void simulate_read_error_metrics(model::metrics::run_metrics& metrics); + template std::string to_string(const std::vector& values) { @@ -31,12 +37,117 @@ std::string to_string(const std::vector& values) return out.str(); } +/** Setup for tests that compare two run summaries */ +struct imaging_table_tests : public generic_test_fixture< model::table::imaging_table > {}; + +TEST_P(imaging_table_tests, compare_to_baseline) +{ + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + ASSERT_EQ(expected.column_count(), actual.column_count()) << "Baseline: " << to_string(expected.columns()) << "\n Actual: " << to_string(actual.columns()); + ASSERT_EQ(expected.row_count(), actual.row_count()); + EXPECT_THAT(expected.columns(), ::testing::ElementsAreArray(actual.columns())); + std::vector expected_idx(expected.row_count(), expected.row_count()); + for(size_t i=0;i actual_idx(expected_idx); + + + // Account for differences in rounding due to writing floating-point numbers as text (and double/float issues) + // C# rounds to nearest even + // InterOp rounds using std::floor(val+0.5) + const float round_tol = 0.101f; + std::vector round_tols(logic::table::max_digits()+1); + round_tols[0] = 1e-5f; + round_tols[1] = round_tol; + for(size_t i=2;i> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "imaging_write_read_generator"; + } +}; + +imaging_table_tests::generator_type imaging_io_generators[] = { + wrap(new standard_parameter_generator(0)), +}; + +// Setup unit tests for extended_tile_metrics_tests +INSTANTIATE_TEST_CASE_P(imaging_unit_tests, + imaging_table_tests, + ::testing::ValuesIn(imaging_io_generators)); //--------------------------------------------------------------------------------------------------------------------- // Regression test section //--------------------------------------------------------------------------------------------------------------------- -/** Imaging regression test fixture. +/** Imaging regression test fixture * * This provides the test group name, provides the tested type as imaging_table, ensures the output file is prefixed with * image_table and finally properly sets up the expected imaging_table object. @@ -45,7 +156,16 @@ class imaging_table_regression_test : public abstract_regression_test_generator< { typedef abstract_regression_test_generator< model::table::imaging_table > parent_t; public: + /** Constructor + * + * @param test_dir sub directory for baseline + */ imaging_table_regression_test(const std::string& test_dir) : parent_t(test_dir){} + /** Constructor + * + * @param run_folder run folder + * @param test_dir sub directory for baseline + */ imaging_table_regression_test(const std::string& run_folder, const std::string& test_dir) : parent_t(run_folder, test_dir){} protected: @@ -53,22 +173,29 @@ class imaging_table_regression_test : public abstract_regression_test_generator< * * @param baseline_file baseline file * @param expected expected model data + * @return true if the file was found, and the read completed without failure */ - void read_expected(const std::string& baseline_file, model::table::imaging_table& expected)const + bool read_expected(const std::string& baseline_file, model::table::imaging_table& expected)const { std::ifstream fin(baseline_file.c_str()); + if( !fin.good() ) return false; fin >> expected; + if( fin.eof() ) return true; + return !fin.fail(); } /** Read the actual data from the run folder * * @param run_folder run folder * @param actual actual model data + * @return true if data was generated */ - void generate_actual(const std::string& run_folder, model::table::imaging_table& actual)const + bool generate_actual(const std::string& run_folder, model::table::imaging_table& actual)const { model::metrics::run_metrics actual_metrics; actual_metrics.read(run_folder); + if( actual_metrics.empty() ) return false; logic::table::create_imaging_table(actual_metrics, actual); + return actual.column_count()*actual.row_count() > 0; } /** Write the actual data to the run folder * @@ -85,9 +212,18 @@ class imaging_table_regression_test : public abstract_regression_test_generator< * * @return pointer to new copy */ - base_type clone()const + base_t clone()const { - return new imaging_table_regression_test(m_run_folder, m_test_dir); + return new imaging_table_regression_test(*this); + } + /** Create a copy of the current object with the given run folder + * + * @param run_folder run folder + * @return pointer to new copy + */ + base_t clone(const std::string& run_folder)const + { + return new imaging_table_regression_test(run_folder, m_test_dir); } /** Write generator info to output stream * @@ -98,56 +234,6 @@ class imaging_table_regression_test : public abstract_regression_test_generator< out << "imaging_table_regression_test - " << io::basename(m_run_folder); } }; -/** Setup for tests that compare two run summaries */ -struct imaging_table_tests : public generic_test_fixture< model::table::imaging_table > {}; - -TEST_P(imaging_table_tests, compare_to_baseline) -{ - if(!test) return; - ASSERT_EQ(expected.column_count(), actual.column_count()) << "Baseline: " << to_string(expected.columns()) << "\n Actual: " << to_string(actual.columns()); - ASSERT_EQ(expected.row_count(), actual.row_count()); - EXPECT_THAT(expected.columns(), ::testing::ElementsAreArray(actual.columns())); - std::vector expected_idx(expected.row_count(), expected.row_count()); - for(size_t i=0;i actual_idx(expected_idx); - - - // Account for differences in rounding due to writing floating-point numbers as text (and double/float issues) - // C# rounds to nearest even - // InterOp rounds using std::floor(val+0.5) - const float round_tol = 0.101f; - const float round_tols[] = {1e-5f, round_tol, round_tol/10, round_tol/100}; - -// @todo remove the following sort - std::stable_sort(expected_idx.begin(), expected_idx.end(), model::table::imaging_table_id_less(expected)); - std::stable_sort(actual_idx.begin(), actual_idx.end(), model::table::imaging_table_id_less(actual)); - for(size_t row_index=0;row_index +#include "src/tests/interop/logic/inc/metric_filter_iterator.h" +#include "src/tests/interop/inc/generic_fixture.h" + +namespace illumina { namespace interop { namespace unittest +{ + /** Generate the plot from an empty metric set + * + * The expected plot is empty + */ + template + class empty_plot_test_generator : public abstract_generator + { + typedef typename abstract_generator::parent_type base_type; + public: + /** Constructor + * + * @param plot_type type of the plot + */ + empty_plot_test_generator(const constants::plot_types plot_type=constants::UnknownPlotType) : + m_info(model::run::flowcell_layout(1, 2)), + m_plot_type(plot_type) + { + std::string channels[] = {"Red", "Green"}; + m_info.channels(util::to_vector(channels)); + m_info.set_naming_method(constants::FourDigit); + const model::run::read_info reads[] = {model::run::read_info(1, 1, 3, false)}; + m_info.reads(util::to_vector(reads)); + if(plot_type != constants::UnknownPlotType) m_metric_iterator.reset(m_info, plot_type); + } + /** Clone the concrete implementation TODO: Remove + * + * @param name run folder + * @return copy of this object + */ + base_type operator()(const constants::plot_types plot_type)const + { + return new empty_plot_test_generator(plot_type); + } + + /** Generate the expected and actual metric sets + * + * @note expected plot data is empty + * @param actual actual plot data + */ + ::testing::AssertionResult generate(PlotData&, PlotData &actual, bool*) const + { + model::metrics::run_metrics metrics; + metrics.run_info(m_info); + m_metric_iterator.plot(metrics, actual); + return ::testing::AssertionSuccess(); + } + + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + base_type clone() const + { + return new empty_plot_test_generator(*this); + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << constants::to_string(m_plot_type); + out << "Metric_" << constants::to_string(m_metric_iterator.metric()) << "_"; + out << m_metric_iterator.options(); + } + /** Advance to the next type + * + * @return true when the generator has finished, and the next parameter can be obtained + */ + bool advance() + { + return m_metric_iterator.advance(); + } + + private: + metric_filter_iterator m_metric_iterator; + model::run::info m_info; + constants::plot_types m_plot_type; + }; + + +}}} + diff --git a/src/tests/interop/logic/inc/metric_filter_iterator.h b/src/tests/interop/logic/inc/metric_filter_iterator.h new file mode 100644 index 000000000..f168c07c9 --- /dev/null +++ b/src/tests/interop/logic/inc/metric_filter_iterator.h @@ -0,0 +1,266 @@ +/** Iterator over all possible filters and metrics + * + * + * @file + * @date 10/27/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include "interop/model/run/info.h" +#include "interop/model/plot/filter_options.h" +#include "interop/logic/plot/plot_by_cycle.h" +#include "interop/logic/plot/plot_by_lane.h" +#include "interop/logic/plot/plot_flowcell_map.h" +#include "interop/logic/plot/plot_qscore_heatmap.h" +#include "interop/logic/plot/plot_qscore_histogram.h" +#include "interop/logic/plot/plot_sample_qc.h" + +namespace illumina { namespace interop { namespace unittest +{ + /** Iterator over metric types and filter options + */ + class metric_filter_iterator + { + typedef std::vector metric_type_vector_t; + typedef metric_type_vector_t::const_iterator const_metric_type_iterator_t; + public: + /** Constructor */ + metric_filter_iterator() : m_options(constants::UnknownTileNamingMethod), + m_current_metric(m_metric_types.begin()), + m_plot_type(constants::UnknownPlotType) + { + } + + public: + /** Copy constructor + * + * @param source source iterator to copy + */ + metric_filter_iterator(const metric_filter_iterator& source) : + m_metric_types(source.m_metric_types), + m_options(source.m_options), + m_run_info(source.m_run_info), + m_plot_type(source.m_plot_type) + { + m_current_metric = m_metric_types.begin() + std::distance(source.m_metric_types.begin(), source.m_current_metric); + m_current_filter = m_options.option_iterator(m_run_info, *m_current_metric, m_plot_type, true); + } + + public: + /** Reset iterator ranges and set to start + * + * @param run_info_filename path to run info xml + * @param plot_type type of the plot + */ + void reset(const std::string &run_info_filename, const constants::plot_types plot_type) + { + model::metrics::run_metrics run; + try + { + run.read(run_info_filename); + } + catch (const std::exception &){} + reset(run.run_info(), plot_type); + } + /** Plot a run by lane or cycle + * + * @param run run metrics + * @param data candle stick plot data + */ + void plot(model::metrics::run_metrics& run, model::plot::plot_data& data)const + { + model::plot::plot_data unused; + switch(m_plot_type) + { + case constants::ByCyclePlot: + logic::plot::plot_by_cycle(run, *m_current_metric, m_options, data); + break; + case constants::ByLanePlot: + logic::plot::plot_by_lane(run, *m_current_metric, m_options, data); + break; + default: + INTEROP_THROW(std::runtime_error, "Plot type unsupported"); + }; + } + /** Plot a histogram of the q-scores + * + * @param run run metrics + * @param data bar plot data + */ + void plot(model::metrics::run_metrics& run, model::plot::plot_data& data)const + { + switch(m_plot_type) + { + case constants::QHistogramPlot: + logic::plot::plot_qscore_histogram(run, m_options, data); + break; + case constants::SampleQCPlot: + logic::plot::plot_sample_qc(run, m_options.lane(), data); + break; + default: + INTEROP_THROW(std::runtime_error, "Plot type unsupported"); + }; + } + /** Plot a q-score heat map of the run + * + * @param run run metrics + * @param data heatmap plot data + */ + void plot(model::metrics::run_metrics& run, model::plot::heatmap_data& data)const + { + switch(m_plot_type) + { + case constants::QHeatmapPlot: + logic::plot::plot_qscore_heatmap(run, m_options, data); + break; + default: + INTEROP_THROW(std::runtime_error, "Plot type unsupported"); + }; + } + /** Plot a flow cell of the run + * + * @param run run metrics + * @param data flowcell plot data + */ + void plot(model::metrics::run_metrics& run, model::plot::flowcell_data& data)const + { + switch(m_plot_type) + { + case constants::FlowcellPlot: + logic::plot::plot_flowcell_map(run, *m_current_metric, m_options, data); + break; + default: + INTEROP_THROW(std::runtime_error, "Plot type unsupported"); + }; + } + + /** Reset iterator ranges and set to start + * + * @param run_info run info + * @param plot_type type of the plot + */ + void reset(const model::run::info &run_info, const constants::plot_types plot_type) + { + m_run_info = run_info; + m_plot_type = plot_type; + + m_metric_types.clear(); + switch(plot_type) + { + case constants::ByCyclePlot: + logic::plot::list_by_cycle_metrics(m_metric_types); + break; + case constants::ByLanePlot: + logic::plot::list_by_lane_metrics(m_metric_types); + break; + case constants::FlowcellPlot: + logic::plot::list_flowcell_metrics(m_metric_types); + break; + case constants::QHeatmapPlot: + case constants::QHistogramPlot: + m_metric_types.push_back(logic::utils::metric_type_description_t(constants::Q30Percent, "")); + break; + case constants::SampleQCPlot: + // Sentinel (turn off surface filtering) + m_metric_types.push_back(logic::utils::metric_type_description_t(constants::UnknownMetricType, "")); + break; + default: + INTEROP_THROW(std::runtime_error, "Plot type unsupported"); + }; + + // Reduce the number of filters! + model::run::flowcell_layout layout = run_info.flowcell(); + layout.lane_count(1); + layout.swath_count(1); + layout.tile_count(1); + m_run_info.flowcell(layout); + std::string channels[] = {"Red", "Green"}; + m_run_info.channels(util::to_vector(channels)); + const model::run::read_info reads[] = {model::run::read_info(1, 1, 1, false)}; + m_run_info.reads(util::to_vector(reads)); + + m_plot_type = plot_type; + m_options.tile_naming_method(m_run_info.flowcell().naming_method()); + m_current_metric = m_metric_types.begin(); + if (m_current_metric == m_metric_types.end()) + INTEROP_THROW(std::runtime_error, "Metric iterator started exhausted"); + m_current_filter = m_options.option_iterator(m_run_info, *m_current_metric, plot_type); + } + /** Get current metric + * + * @return current metric + */ + constants::metric_type metric() const + { + if (m_current_metric == m_metric_types.end()) + INTEROP_THROW(std::runtime_error, "Metric iterator started exhausted"); + return *m_current_metric; + } + + /** Get filter options + * + * @return filter options + */ + const model::plot::filter_options &options() const + { + return m_options; + } + + /** Advance to the next type + * + * @return true when the generator has finished, and the next parameter can be obtained + */ + bool advance() + { + if (m_current_metric == m_metric_types.end()) + return true; + ++m_current_filter; + if (m_current_filter.is_done()) + { + ++m_current_metric; + if( m_current_metric != m_metric_types.end()) + m_current_filter = m_options.option_iterator(m_run_info, *m_current_metric, m_plot_type); + } + return m_current_metric == m_metric_types.end(); + } + /** Test if the iterator is exhausted + * + * @return true when the generator has finished + */ + bool is_done() + { + return m_current_metric == m_metric_types.end(); + } + /** Get a vector of metric types + * + * @return vector of metric types + */ + metric_type_vector_t& metric_types() + { + return m_metric_types; + } + + protected: + /** Reset the metric iterator */ + void reset() + { + m_current_metric = m_metric_types.begin(); + } + + protected: + /** Vector of metrics */ + metric_type_vector_t m_metric_types; + /** Filter options */ + model::plot::filter_options m_options; + private: + util::chain_range_iterator m_current_filter; + const_metric_type_iterator_t m_current_metric; + model::run::info m_run_info; + constants::plot_types m_plot_type; + }; + +}}} + + diff --git a/src/tests/interop/logic/inc/plot_regression_test_generator.h b/src/tests/interop/logic/inc/plot_regression_test_generator.h new file mode 100644 index 000000000..0e24057fd --- /dev/null +++ b/src/tests/interop/logic/inc/plot_regression_test_generator.h @@ -0,0 +1,177 @@ +/** Generate plots for regression testing + * + * + * @file + * @date 11/3/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include "src/tests/interop/logic/inc/metric_filter_iterator.h" +#include "src/tests/interop/inc/abstract_regression_test_generator.h" + +namespace illumina { namespace interop { namespace unittest +{ + + /** Plot regression test fixture + * + */ + template + class plot_regression_test_generator : public abstract_regression_test_generator< PlotData > + { + typedef abstract_regression_test_generator< PlotData > parent_t; + typedef typename parent_t::base_t base_type; + public: + /** Constructor + * + * @param test_dir sub directory containing the tests + * @param plot_type type of the plot + */ + plot_regression_test_generator(const std::string& test_dir, const constants::plot_types plot_type) : + parent_t(test_dir), m_plot_type(plot_type) + { + //regression_test_data::instance().add_subdir(test_dir); + } + /** Constructor + * + * @param run_folder run folder + * @param test_dir sub directory containing the tests + * @param plot_type type of the plot + */ + plot_regression_test_generator(const std::string& run_folder, + const std::string& test_dir, + const constants::plot_types plot_type) : + parent_t(run_folder, test_dir), m_plot_type(plot_type) + { + m_metric_iterator.reset(parent_t::m_run_folder, plot_type); + } + + public: + /** Get the full path of the baseline output file + * + * @return full path + */ + std::string baseline()const + { + return parent_t::baseline()+"_"+name(); + } + + protected: + /** Read the actual data from the run folder + * + * @param run_folder run folder + * @param actual actual model data + * @return true if data was generated + */ + bool generate_actual(const std::string& run_folder, PlotData& actual)const + { + model::metrics::run_metrics& actual_metrics = get_metrics(run_folder); + if( actual_metrics.empty() ) return false; + try + { + m_metric_iterator.plot(actual_metrics, actual); + } + catch(const std::exception& ex) + { + std::cerr << "generate_actual: " << ex.what() << std::endl; + throw; + } + return !actual.empty(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream& out)const + { + out << name(); + } + /** Advance to the next type + * + * @return true when the generator has finished, and the next parameter can be obtained + */ + bool advance() + { + return m_metric_iterator.advance(); + } + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + base_type clone() const + { + return new plot_regression_test_generator(*this); + } + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + base_type clone(const std::string& run_folder) const + { + return new plot_regression_test_generator(run_folder, parent_t::m_test_dir, m_plot_type); + } + /** Read the expected data from the baseline file into the model + * + * @param baseline_file baseline file + * @param expected expected model data + * @return true if the file was found, and the read completed without failure + */ + bool read_expected(const std::string& baseline_file, PlotData& expected)const + { + std::ifstream fin(baseline_file.c_str()); + if( !fin.good() ) return false; + try + { + fin >> expected; + } + catch(const std::exception& ex) + { + std::cerr << "Failed with exception: " << ex.what() << std::endl; + return false; + } + if( fin.eof() ) return !expected.empty(); + return !fin.fail() && !expected.empty(); + } + /** Write the actual data to the run folder + * + * @param baseline_file baseline file + * @param actual actual model data + */ + bool write_actual(const std::string& baseline_file, const PlotData& actual)const + { + std::ofstream fout(baseline_file.c_str()); + fout << actual; + return fout.good(); + } + + private: + static model::metrics::run_metrics& get_metrics(const std::string& run_folder) + { + static std::string current; + static model::metrics::run_metrics actual_metrics; + if( current != run_folder ) + { + current = run_folder; + actual_metrics.read(run_folder); + } + + return actual_metrics; + } + + private: + std::string name()const + { + return constants::to_string(m_plot_type) + "_" + + io::basename(parent_t::m_run_folder) + + "_Metric_" + constants::to_string(m_metric_iterator.metric()) + + "_" + util::lexical_cast(m_metric_iterator.options()); + } + + private: + constants::plot_types m_plot_type; + metric_filter_iterator m_metric_iterator; + }; + + +}}} diff --git a/src/tests/interop/logic/index_summary_test.cpp b/src/tests/interop/logic/index_summary_test.cpp new file mode 100644 index 000000000..199246a5d --- /dev/null +++ b/src/tests/interop/logic/index_summary_test.cpp @@ -0,0 +1,364 @@ +/** Unit tests for the index summary + * + * + * @file + * @date 11/14/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include "interop/logic/summary/index_summary.h" +#include "src/tests/interop/metrics/inc/index_metrics_test.h" +#include "src/tests/interop/metrics/inc/tile_metrics_test.h" +#include "src/tests/interop/inc/abstract_regression_test_generator.h" + + +using namespace illumina::interop::model::summary; +using namespace illumina::interop; +using namespace illumina::interop::unittest; +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::io; + + +/** Setup for tests that compare two index summaries */ +struct index_summary_tests : public generic_test_fixture {}; + +/** Test if calculated expected index flowcell summary matches the actual index flowcell summary + */ +TEST_P(index_summary_tests, index_lane_summary) +{ + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + const float tol = 1e-7f; + ASSERT_EQ(actual.size(), expected.size()); + for(size_t lane=0;lane < actual.size();++lane) + { + ::testing::Message trace_message_lane; + trace_message_lane << "Lane Index: " << lane << " of " << actual.size(); + SCOPED_TRACE(trace_message_lane); + const index_lane_summary& actual_lane_summary = actual[lane]; + const index_lane_summary& expected_lane_summary = expected[lane]; + EXPECT_EQ(expected_lane_summary.total_reads(), actual_lane_summary.total_reads()); + EXPECT_EQ(expected_lane_summary.total_pf_reads(), actual_lane_summary.total_pf_reads()); + INTEROP_EXPECT_NEAR(expected_lane_summary.total_fraction_mapped_reads(), + actual_lane_summary.total_fraction_mapped_reads(), + tol); + INTEROP_EXPECT_NEAR(expected_lane_summary.mapped_reads_cv(), actual_lane_summary.mapped_reads_cv(), tol); + INTEROP_EXPECT_NEAR(expected_lane_summary.min_mapped_reads(), actual_lane_summary.min_mapped_reads(), tol); + INTEROP_EXPECT_NEAR(expected_lane_summary.max_mapped_reads(), actual_lane_summary.max_mapped_reads(), tol); + ASSERT_EQ(expected_lane_summary.size(), actual_lane_summary.size()); + for(size_t index=0;index reads(1, model::run::read_info(1, 1, 3, false)); + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + model::run::info run_info("XX", + "", + 1, + model::run::flowcell_layout(lane_count), + channels, + model::run::image_dimensions(), + reads); + + model::summary::index_flowcell_summary expected; + index_metric_v1::create_summary(expected); + model::summary::index_flowcell_summary actual; + model::metrics::run_metrics metrics(run_info); + try + { + std::string data; + index_metric_v1::create_binary_data(data); + io::read_interop_from_string(data, + metrics.get()); + tile_metric_v2::create_binary_data(data); + io::read_interop_from_string(data, + metrics.get()); + } + catch (const std::exception &) + {} + logic::summary::summarize_index_metrics(metrics, actual); + + const float tol = 1e-7f; + ASSERT_EQ(expected.size(), actual.size()); + for (size_t lane = 0; lane < expected.size(); ++lane) + { + const model::summary::index_lane_summary &expected_lane = expected[lane]; + const model::summary::index_lane_summary &actual_lane = actual[lane]; + EXPECT_EQ(expected_lane.total_reads(), actual_lane.total_reads()); + EXPECT_EQ(expected_lane.total_pf_reads(), actual_lane.total_pf_reads()); + INTEROP_EXPECT_NEAR(expected_lane.total_fraction_mapped_reads(), actual_lane.total_fraction_mapped_reads(), tol); + INTEROP_EXPECT_NEAR(expected_lane.mapped_reads_cv(), + actual_lane.mapped_reads_cv(), + tol); + INTEROP_EXPECT_NEAR(expected_lane.min_mapped_reads(), actual_lane.min_mapped_reads(), tol); + INTEROP_EXPECT_NEAR(expected_lane.max_mapped_reads(), actual_lane.max_mapped_reads(), tol); + ASSERT_EQ(expected_lane.size(), actual_lane.size()); + for (size_t index = 0; index < expected_lane.size(); ++index) + { + const model::summary::index_count_summary &expected_count = expected_lane[index]; + const model::summary::index_count_summary &actual_count = actual_lane[index]; + EXPECT_EQ(expected_count.id(), actual_count.id()); + EXPECT_EQ(expected_count.index1(), actual_count.index1()); + EXPECT_EQ(expected_count.index2(), actual_count.index2()); + EXPECT_EQ(expected_count.cluster_count(), actual_count.cluster_count()); + EXPECT_EQ(expected_count.sample_id(), actual_count.sample_id()); + EXPECT_EQ(expected_count.project_name(), actual_count.project_name()); + INTEROP_EXPECT_NEAR(expected_count.fraction_mapped(), actual_count.fraction_mapped(), tol); + } + } +} + +//--------------------------------------------------------------------------------------------------------------------- +// Unit test section +//--------------------------------------------------------------------------------------------------------------------- + +/** Generate the actual metric set by reading in from hardcoded binary buffer + * + * The expected metric set is provided by the generator. + */ +template +class index_summary_generator : public abstract_generator +{ + typedef typename IndexGen::metric_set_t index_metric_set_t; + typedef typename TileGen::metric_set_t tile_metric_set_t; + typedef abstract_generator::parent_type base_t; +public: + /** Generate the expected and actual metric sets + * + * @param expected expected metric set + * @param actual actual metric set + */ + ::testing::AssertionResult generate(model::summary::index_flowcell_summary &expected, + model::summary::index_flowcell_summary &actual, + bool*) const + { + expected.clear(); + actual.clear(); + const size_t lane_count = 8; + const size_t surface_count = 2; + const size_t swath_count = 4; + const size_t tile_count = 99; + const size_t sections_per_lane = 6; + const size_t lanes_per_section = 6; + IndexGen::create_summary(expected); + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + + std::vector reads; + reads.reserve(3); + reads.push_back(model::run::read_info(1, 1, 3, false)); + reads.push_back(model::run::read_info(2, 4, 10, false)); + reads.push_back(model::run::read_info(3, 11, 15, false)); + actual = model::summary::index_flowcell_summary(); + model::run::info run_info("XX", + "", + 1, + model::run::flowcell_layout(lane_count, + surface_count, + swath_count, + tile_count, + sections_per_lane, + lanes_per_section), + channels, + model::run::image_dimensions(), + reads); + run_info.set_naming_method(constants::FourDigit); + model::metrics::run_metrics metrics(run_info); + IndexGen::create_expected(metrics.get()); + TileGen::create_expected(metrics.get()); + metrics.finalize_after_load(); + logic::summary::summarize_index_metrics(metrics, actual); + return ::testing::AssertionSuccess(); + } + + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + base_t clone() const + { + return new index_summary_generator; + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "index_summary_generator<" << IndexGen::name() << "_" << TileGen::name() << ">"; + } +}; + +/** Generator to test writing and reading an index summary */ +class index_write_read_generator +{ +public: + /** Define parameter type */ + typedef int parameter_type; +public: + /** Constructor */ + index_write_read_generator(const int){} + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(model::summary::index_flowcell_summary& expected, + model::summary::index_flowcell_summary &actual) const + { + index_metric_v1::create_summary(expected); + std::ostringstream oss; + oss << expected; + + std::istringstream iss(oss.str()); + iss >> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "index_write_read_generator"; + } +}; + +index_summary_tests::generator_type index_summary_unit_test_generators[] = { + wrap(new index_summary_generator()), + wrap(new standard_parameter_generator(0)) +}; + + +// Setup unit tests for run summary tests +INSTANTIATE_TEST_CASE_P(index_summary_unit_test, + index_summary_tests, + ::testing::ValuesIn(index_summary_unit_test_generators)); + +//--------------------------------------------------------------------------------------------------------------------- +// Regression test section +//--------------------------------------------------------------------------------------------------------------------- + +/** Generate the actual run summary writing out the expected and reading it back in again + */ +class regression_test_index_summary_generator : public abstract_regression_test_generator +{ + typedef abstract_regression_test_generator parent_t; +public: + /** Constructor + * + * @param test_dir sub folder where tests are stored + */ + regression_test_index_summary_generator(const std::string &test_dir) : parent_t(test_dir) + {} + + /** Constructor + * + * @param run_folder run folder with data + * @param test_dir sub folder where tests are stored + */ + regression_test_index_summary_generator(const std::string &run_folder, const std::string &test_dir) : + parent_t(run_folder,test_dir) + {} + +protected: + /** Read the expected data from the baseline file into the model + * + * @param baseline_file baseline file + * @param expected expected model data + * @return true if the file was found, and the read completed without failure + */ + bool read_expected(const std::string &baseline_file, model::summary::index_flowcell_summary &expected) const + { + std::ifstream fin(baseline_file.c_str(), std::ifstream::binary); + if( !fin.good() ) return false; + fin >> expected; + if( fin.eof() ) return true; + return !fin.fail(); + } + + /** Read the actual data from the run folder + * + * @param run_folder run folder + * @param actual actual model data + * @return true if data was generated + */ + bool generate_actual(const std::string &run_folder, model::summary::index_flowcell_summary &actual) const + { + model::metrics::run_metrics actual_metrics; + actual_metrics.read(run_folder); + if( actual_metrics.empty() ) return false; + logic::summary::summarize_index_metrics(actual_metrics, actual); + return actual.size() > 0; + } + + /** Write the actual data to the run folder + * + * @param baseline_file baseline file + * @param actual actual model data + */ + bool write_actual(const std::string &baseline_file, const model::summary::index_flowcell_summary &actual) const + { + std::ofstream fout(baseline_file.c_str(), std::ofstream::binary); + fout << actual; + return fout.good(); + } + /** Create a copy of the current object with the given run folder + * + * @param run_folder run folder + * @return pointer to new copy + */ + base_t clone(const std::string& run_folder)const + { + return new regression_test_index_summary_generator(run_folder, m_test_dir); + } + + /** Create a copy of the current object + * + * @return pointer to new copy + */ + base_t clone() const + { + return new regression_test_index_summary_generator(*this); + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "regression_test_index_summary_generator - " << io::basename(m_run_folder); + } +}; + +regression_test_index_summary_generator index_summary_regression_gen("index_summary"); + +INSTANTIATE_TEST_CASE_P(index_summary_regression_test, + index_summary_tests, + ProxyValuesIn(index_summary_regression_gen, regression_test_data::instance().files())); + + diff --git a/src/tests/interop/logic/plot_bar_test.cpp b/src/tests/interop/logic/plot_bar_test.cpp new file mode 100644 index 000000000..7d506b6c8 --- /dev/null +++ b/src/tests/interop/logic/plot_bar_test.cpp @@ -0,0 +1,154 @@ +/** Unit tests for bar type plot logic + * + * + * @file + * @date 11/2/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include +#include +#include "interop/model/plot/bar_point.h" +#include "interop/model/plot/plot_data.h" +#include "interop/model/plot/filter_options.h" +#include "src/tests/interop/logic/inc/empty_plot_test_generator.h" +#include "src/tests/interop/logic/inc/plot_regression_test_generator.h" + +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::plot; +using namespace illumina::interop::io; +using namespace illumina::interop; +using namespace illumina::interop::unittest; + + +/** Define the plot data */ +typedef plot_data bar_plot_data; +/** Setup for tests that compare two bar plots */ +struct bar_plot_tests : public generic_test_fixture {}; + + +/** Test if bar plot matchs expected data + */ +TEST_P(bar_plot_tests, plot_data) +{ + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + const float tol = 1e-4f; + EXPECT_EQ(actual.title(), expected.title()); + EXPECT_EQ(actual.x_axis().min(), expected.x_axis().min()); + EXPECT_EQ(actual.x_axis().max(), expected.x_axis().max()); + EXPECT_EQ(actual.x_axis().label(), expected.x_axis().label()); + EXPECT_EQ(actual.y_axis().min(), expected.y_axis().min()); + EXPECT_EQ(actual.y_axis().max(), expected.y_axis().max()); + EXPECT_EQ(actual.y_axis().label(), expected.y_axis().label()); + ASSERT_EQ(actual.size(), expected.size()); + for(size_t i=0;i plot_bar_gen; +const constants::plot_types bar_types[] = {constants::QHistogramPlot, constants::SampleQCPlot}; +std::vector plot_bar_gen_data(bar_types, + bar_types+util::length_of(bar_types)); +// Setup unit tests for run summary tests +INSTANTIATE_TEST_CASE_P(bar_plot_unit_test, + bar_plot_tests, + ProxyValuesIn(plot_bar_gen, plot_bar_gen_data)); + + +class bar_write_read_generator +{ +public: + /** Define parameter type */ + typedef int parameter_type; +public: + /** Constructor */ + bar_write_read_generator(const int){} + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(bar_plot_data& expected, bar_plot_data &actual) const + { + expected.set_title("Test Plot"); + expected.set_xlabel("X-axis"); + expected.set_ylabel("Y-axis"); + expected.set_range(0, 1, 0, 2); + expected.resize(2); + expected.push_back(series("Q Score", "DarkGreen", model::plot::series::Bar)); + expected.push_back(model::plot::series("Threshold(>=Q30)", "Green", model::plot::series::Line)); + expected[0].resize(3); + expected[1].resize(3); + expected[0][0].set(std::numeric_limits::quiet_NaN(), 1, 2); + expected[0][1].set(3, 4, 5); + expected[0][2].set(6, 7, 8); + expected[1][0].set(9, 10, 11); + expected[1][1].set(12, 13, 14); + expected[1][2].set(15, 16, 17); + expected[0].add_option("Dummy1"); + + + std::ostringstream oss; + oss << expected; + + std::istringstream iss(oss.str()); + iss >> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "bar_write_read_generator"; + } +}; + +bar_plot_tests::generator_type plot_bar_generators[] = { + wrap(new standard_parameter_generator(0)), +}; + +// Setup unit tests for extended_tile_metrics_tests +INSTANTIATE_TEST_CASE_P(bar_plot_add_unit_tests, + bar_plot_tests, + ::testing::ValuesIn(plot_bar_generators)); + +//--------------------------------------------------------------------------------------------------------------------- +// Regression test section +//--------------------------------------------------------------------------------------------------------------------- + + +plot_regression_test_generator qscore_histogram_regression_test_gen("plot", constants::QHistogramPlot); + +INSTANTIATE_TEST_CASE_P(qscore_histogram_regression_test, + bar_plot_tests, + ProxyValuesIn(qscore_histogram_regression_test_gen, regression_test_data::instance().files())); + + +plot_regression_test_generator sample_qc_regression_test_gen("plot", constants::QHistogramPlot); + +INSTANTIATE_TEST_CASE_P(sample_qc_regression_test, + bar_plot_tests, + ProxyValuesIn(sample_qc_regression_test_gen, regression_test_data::instance().files())); + diff --git a/src/tests/interop/logic/plot_candle_stick_test.cpp b/src/tests/interop/logic/plot_candle_stick_test.cpp new file mode 100644 index 000000000..9adee611a --- /dev/null +++ b/src/tests/interop/logic/plot_candle_stick_test.cpp @@ -0,0 +1,182 @@ +/** Unit tests for candle stick type plot logic + * + * + * @file + * @date 10/26/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include +#include +#include "interop/model/plot/candle_stick_point.h" +#include "interop/model/plot/plot_data.h" +#include "interop/model/plot/filter_options.h" +#include "src/tests/interop/metrics/inc/error_metrics_test.h" +#include "src/tests/interop/metrics/inc/extraction_metrics_test.h" +#include "src/tests/interop/logic/inc/empty_plot_test_generator.h" +#include "src/tests/interop/logic/inc/plot_regression_test_generator.h" + +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::plot; +using namespace illumina::interop::io; +using namespace illumina::interop; +using namespace illumina::interop::unittest; + +/** Define the plot data */ +typedef plot_data candle_stick_plot_data; +/** Setup for tests that compare two candle stick plots */ +struct candle_stick_tests : public generic_test_fixture {}; + +/** Test that the filter iterator works */ +TEST(candle_stick_tests, test_filter_iterator_by_cycle) +{ + const std::string channels[] = {"Red", "Green"}; + const model::run::read_info reads[] = {model::run::read_info(1, 3, false)}; + model::run::info run_info(model::run::flowcell_layout(2, 2), util::to_vector(channels), util::to_vector(reads)); + run_info.set_naming_method(constants::FourDigit); + metric_filter_iterator metric_iterator; + metric_iterator.reset(run_info, constants::ByCyclePlot); + while(!metric_iterator.is_done()) + { + ASSERT_NO_THROW(metric_iterator.options().validate(metric_iterator.metric(), run_info, true)); + if(metric_iterator.advance()) break; + } +} + +/** Test if candle stick plot matchs expected data + */ +TEST_P(candle_stick_tests, plot_data) +{ + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + const float tol = 1e-4f; + EXPECT_EQ(actual.title(), expected.title()); + EXPECT_EQ(actual.x_axis().min(), expected.x_axis().min()); + EXPECT_EQ(actual.x_axis().max(), expected.x_axis().max()) << actual.empty() << ", " << expected.empty(); + EXPECT_EQ(actual.x_axis().label(), expected.x_axis().label()); + EXPECT_EQ(actual.y_axis().min(), expected.y_axis().min()); + EXPECT_EQ(actual.y_axis().max(), expected.y_axis().max()); + EXPECT_EQ(actual.y_axis().label(), expected.y_axis().label()); + ASSERT_EQ(actual.size(), expected.size()); + for(size_t i=0;i plot_by_cycle_gen; +const constants::plot_types candle_stick_types[] = {constants::ByCyclePlot, + constants::ByLanePlot}; +std::vector plot_by_cycle_gen_data(candle_stick_types, + candle_stick_types+util::length_of(candle_stick_types)); +// Setup unit tests for run summary tests +INSTANTIATE_TEST_CASE_P(candle_stick_unit_test, + candle_stick_tests, + ProxyValuesIn(plot_by_cycle_gen, plot_by_cycle_gen_data)); + +class candle_stick_read_generator +{ +public: + /** Define parameter type */ + typedef int parameter_type; +public: + /** Constructor */ + candle_stick_read_generator(const int){} + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(candle_stick_plot_data& expected, candle_stick_plot_data &actual) const + { + expected.set_title("Test Plot"); + expected.set_xlabel("X-axis"); + expected.set_ylabel("Y-axis"); + expected.set_range(0, 1, 0, 2); + expected.resize(2); + expected.push_back(series("Q Score", + "DarkGreen", + model::plot::series::Candlestick)); + expected.push_back(model::plot::series("Threshold(>=Q30)", + "Green", + model::plot::series::Line)); + expected[0].resize(3); + expected[1].resize(3); + expected[0][0] = candle_stick_point(std::numeric_limits::quiet_NaN(), 2, 3, 4, 5, 6, 7, std::vector(1, 44)); + expected[0][1] = candle_stick_point(11, 12, 31, 41, 51, 61, 71, std::vector(1, 3)); + expected[0][2] = candle_stick_point(12, 22, 23, 24, 25, 26, 27, std::vector(1, 4)); + expected[1][0].set(9, 10); + expected[1][1].set(12, 13); + expected[1][2].set(15, 16); + expected[0].add_option("Dummy1"); + + std::ostringstream oss; + oss << expected; + + std::istringstream iss(oss.str()); + iss >> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "candle_stick_read_generator"; + } +}; + +candle_stick_tests::generator_type plot_candle_stick_generators[] = { + wrap(new standard_parameter_generator(0)), +}; + +// Setup unit tests for extended_tile_metrics_tests +INSTANTIATE_TEST_CASE_P(candle_stick_unit_tests, + candle_stick_tests, + ::testing::ValuesIn(plot_candle_stick_generators)); + +//--------------------------------------------------------------------------------------------------------------------- +// Regression test section +//--------------------------------------------------------------------------------------------------------------------- + + + +plot_regression_test_generator by_cycle_regression_test_gen("plot", constants::ByCyclePlot); + +INSTANTIATE_TEST_CASE_P(plot_by_cycle_regression_test, + candle_stick_tests, + ProxyValuesIn(by_cycle_regression_test_gen, regression_test_data::instance().files())); + + +plot_regression_test_generator by_lane_regression_test_gen("plot", constants::ByLanePlot); + +INSTANTIATE_TEST_CASE_P(plot_by_lane_regression_test, + candle_stick_tests, + ProxyValuesIn(by_lane_regression_test_gen, regression_test_data::instance().files())); + diff --git a/src/tests/interop/logic/plot_flowcell_test.cpp b/src/tests/interop/logic/plot_flowcell_test.cpp new file mode 100644 index 000000000..279de963c --- /dev/null +++ b/src/tests/interop/logic/plot_flowcell_test.cpp @@ -0,0 +1,134 @@ +/** Unit tests for flowcell type plot logic + * + * + * @file + * @date 11/3/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include +#include +#include "interop/model/plot/flowcell_data.h" +#include "interop/model/plot/filter_options.h" +#include "src/tests/interop/logic/inc/empty_plot_test_generator.h" +#include "src/tests/interop/logic/inc/plot_regression_test_generator.h" + + +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::plot; +using namespace illumina::interop::io; +using namespace illumina::interop; +using namespace illumina::interop::unittest; + + +/** Setup for tests that compare two flowcell map plots */ +struct flowcell_plot_tests : public generic_test_fixture {}; + + +/** Test if flowcell matchs expected data + */ +TEST_P(flowcell_plot_tests, plot_data) +{ + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + const float tol = 1e-4f; + EXPECT_EQ(actual.title(), expected.title()); + EXPECT_EQ(actual.subtitle(), expected.subtitle()) << GetParam(); + EXPECT_EQ(actual.x_axis().min(), expected.x_axis().min()); + EXPECT_EQ(actual.x_axis().max(), expected.x_axis().max()); + EXPECT_EQ(actual.x_axis().label(), expected.x_axis().label()); + EXPECT_EQ(actual.y_axis().min(), expected.y_axis().min()); + EXPECT_EQ(actual.y_axis().max(), expected.y_axis().max()); + EXPECT_EQ(actual.y_axis().label(), expected.y_axis().label()); + ASSERT_EQ(actual.row_count(), expected.row_count()); + ASSERT_EQ(actual.column_count(), expected.column_count()); + for(size_t i=0, n=actual.length();i plot_flowcell_gen; +const constants::plot_types flowcell_types[] = {constants::FlowcellPlot}; +std::vector flowcell_gen_data(flowcell_types, + flowcell_types+util::length_of(flowcell_types)); +// Setup unit tests for run summary tests +INSTANTIATE_TEST_CASE_P(plot_flowcell_unit_test, + flowcell_plot_tests, + ProxyValuesIn(plot_flowcell_gen, flowcell_gen_data)); + +class flowcell_write_read_generator +{ +public: + /** Define parameter type */ + typedef int parameter_type; +public: + /** Constructor */ + flowcell_write_read_generator(const int){} + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(flowcell_data& expected, flowcell_data &actual) const + { + expected.set_title("Test Plot"); + expected.set_xlabel("X-axis"); + expected.set_ylabel("Y-axis"); + expected.set_range(0, 1); + expected.resize(2, 3, 1); + size_t i=0; + expected(0,0) = std::numeric_limits::quiet_NaN(); + expected(0,1) = static_cast(i++); + expected(0,2) = static_cast(i++); + expected(1,0) = static_cast(i++); + expected(1,1) = static_cast(i++); + expected(1,2) = static_cast(i++); + expected.tile_id(0,0) = static_cast< ::uint32_t >(i++); + expected.tile_id(0,1) = static_cast< ::uint32_t >(i++); + expected.tile_id(0,2) = static_cast< ::uint32_t >(i++); + expected.tile_id(1,0) = static_cast< ::uint32_t >(i++); + expected.tile_id(1,1) = static_cast< ::uint32_t >(i++); + expected.tile_id(1,2) = static_cast< ::uint32_t >(i++); + std::ostringstream oss; + oss << expected; + + std::istringstream iss(oss.str()); + iss >> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "flowcell_read_generator"; + } +}; + +flowcell_plot_tests::generator_type plot_flowcell_generators[] = { + wrap(new standard_parameter_generator(0)), +}; + +// Setup unit tests for extended_tile_metrics_tests +INSTANTIATE_TEST_CASE_P(flowcell_unit_tests, + flowcell_plot_tests, + ::testing::ValuesIn(plot_flowcell_generators)); + +//--------------------------------------------------------------------------------------------------------------------- +// Regression test section +//--------------------------------------------------------------------------------------------------------------------- + + +plot_regression_test_generator flowcell_regression_test_gen("plot", constants::FlowcellPlot); + +INSTANTIATE_TEST_CASE_P(flowcell_regression_test, + flowcell_plot_tests, + ProxyValuesIn(flowcell_regression_test_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/logic/plot_heatmap_test.cpp b/src/tests/interop/logic/plot_heatmap_test.cpp new file mode 100644 index 000000000..9dcea5d8b --- /dev/null +++ b/src/tests/interop/logic/plot_heatmap_test.cpp @@ -0,0 +1,126 @@ +/** Unit tests for heatmap type plot logic + * + * + * @file + * @date 11/2/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include +#include +#include "interop/model/plot/heatmap_data.h" +#include "interop/model/plot/filter_options.h" +#include "src/tests/interop/logic/inc/empty_plot_test_generator.h" +#include "src/tests/interop/logic/inc/plot_regression_test_generator.h" + + +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::plot; +using namespace illumina::interop::io; +using namespace illumina::interop; +using namespace illumina::interop::unittest; + + +/** Setup for tests that compare two heat map plots */ +struct heatmap_plot_tests : public generic_test_fixture {}; + + +/** Test if Heatmap matchs expected data + */ +TEST_P(heatmap_plot_tests, plot_data) +{ + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + const float tol = 1e-4f; + EXPECT_EQ(actual.title(), expected.title()); + EXPECT_EQ(actual.x_axis().min(), expected.x_axis().min()); + EXPECT_EQ(actual.x_axis().max(), expected.x_axis().max()); + EXPECT_EQ(actual.x_axis().label(), expected.x_axis().label()); + EXPECT_EQ(actual.y_axis().min(), expected.y_axis().min()); + EXPECT_EQ(actual.y_axis().max(), expected.y_axis().max()); + EXPECT_EQ(actual.y_axis().label(), expected.y_axis().label()); + ASSERT_EQ(actual.row_count(), expected.row_count()); + ASSERT_EQ(actual.column_count(), expected.column_count()); + for(size_t i=0, n=actual.length();i plot_heatmap_gen; +const constants::plot_types heatmap_types[] = {constants::QHeatmapPlot}; +std::vector heatmap_gen_data(heatmap_types, + heatmap_types+util::length_of(heatmap_types)); +// Setup unit tests for run summary tests +INSTANTIATE_TEST_CASE_P(plot_heatmap_unit_test, + heatmap_plot_tests, + ProxyValuesIn(plot_heatmap_gen, heatmap_gen_data)); + +class heatmap_write_read_generator +{ +public: + /** Define parameter type */ + typedef int parameter_type; +public: + /** Constructor */ + heatmap_write_read_generator(const int){} + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(heatmap_data& expected, heatmap_data &actual) const + { + expected.set_title("Test Plot"); + expected.set_xlabel("X-axis"); + expected.set_ylabel("Y-axis"); + expected.set_range(0, 1, 0, 2); + expected.resize(2,3); + size_t i=0; + expected(0,0) = std::numeric_limits::quiet_NaN(); + expected(0,1) = static_cast(i++); + expected(0,2) = static_cast(i++); + expected(1,0) = static_cast(i++); + expected(1,1) = static_cast(i++); + expected(1,2) = static_cast(i++); + std::ostringstream oss; + oss << expected; + + std::istringstream iss(oss.str()); + iss >> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "heatmap_read_generator"; + } +}; + +heatmap_plot_tests::generator_type plot_heatmap_generators[] = { + wrap(new standard_parameter_generator(0)), +}; + +// Setup unit tests for extended_tile_metrics_tests +INSTANTIATE_TEST_CASE_P(heatmap_unit_tests, + heatmap_plot_tests, + ::testing::ValuesIn(plot_heatmap_generators)); + +//--------------------------------------------------------------------------------------------------------------------- +// Regression test section +//--------------------------------------------------------------------------------------------------------------------- + + +plot_regression_test_generator qscore_heatmap_regression_test_gen("plot", constants::QHeatmapPlot); + +INSTANTIATE_TEST_CASE_P(qscore_heatmap_regression_test, + heatmap_plot_tests, + ProxyValuesIn(qscore_heatmap_regression_test_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/logic/plot_logic_test.cpp b/src/tests/interop/logic/plot_logic_test.cpp index 2f286997a..7a467ce84 100644 --- a/src/tests/interop/logic/plot_logic_test.cpp +++ b/src/tests/interop/logic/plot_logic_test.cpp @@ -17,43 +17,11 @@ #include "src/tests/interop/metrics/inc/tile_metrics_test.h" #include "src/tests/interop/metrics/inc/q_metrics_test.h" #include "src/tests/interop/metrics/inc/index_metrics_test.h" +#include "src/tests/interop/inc/generic_fixture.h" +#include "src/tests/interop/logic/inc/metric_filter_iterator.h" using namespace illumina::interop; - -TEST(plot_logic, check_plot_by_cycle_list) -{ - std::vector types; - logic::plot::list_by_cycle_metrics(types); - for(size_t i=0;i types; - logic::plot::list_by_lane_metrics(types); - for(size_t i=0;i types; - logic::plot::list_flowcell_metrics(types); - for(size_t i=0;i reads(2); reads[0] = model::run::read_info(1, 1, 26); reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(2, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); + metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(2, 2, 2, 16), + std::vector(), model::run::image_dimensions(), reads)); metrics.set_naming_method(constants::FourDigit); metrics.legacy_channel_update(constants::HiSeq); - unittest::extraction_metric_v2::create_metric_set(metrics.get_set()); + unittest::extraction_metric_v2::create_expected(metrics.get()); model::plot::plot_data data; logic::plot::plot_by_cycle(metrics, constants::Intensity, options, data); - ASSERT_EQ(data.size(), 4u); + const double expected[] = {321, 282.666656, 0, 0}; + ASSERT_EQ(data.size(), channel_count); EXPECT_EQ(data.x_axis().label(), "Cycle"); EXPECT_EQ(data.y_axis().label(), "Intensity"); EXPECT_EQ(data.title(), "All Lanes All Channels"); @@ -96,17 +60,45 @@ TEST(plot_logic, intensity_by_cycle) EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); EXPECT_NEAR(data.x_axis().max(), 3.0f, tol); EXPECT_NEAR(data.y_axis().max(), 353.1f, tol); - for(size_t channel=0;channel<4u;++channel) + size_t k = 0; + for (size_t channel = 0; channel < data.size(); ++channel) { - for(size_t i=0;i (expected[k]), data[channel][i].y(), tol); + k++; } } } +//Check that reading in an empty interop sets values to 0 etc for plot_by_cycle +TEST(plot_logic, intensity_by_cycle_empty_interop) +{ + const float tol = 1e-3f; + model::metrics::run_metrics metrics; + model::plot::filter_options options(constants::FourDigit); + std::vector reads(2); + reads[0] = model::run::read_info(1, 1, 26); + reads[1] = model::run::read_info(2, 27, 76); + metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(2, 2, 2, 16), + std::vector(), model::run::image_dimensions(), reads)); + metrics.set_naming_method(constants::FourDigit); + metrics.legacy_channel_update(constants::HiSeq); + + model::plot::plot_data data; + logic::plot::plot_by_cycle(metrics, constants::Intensity, options, data); + ASSERT_EQ(data.size(), 0u); + EXPECT_EQ(data.x_axis().label(), ""); + EXPECT_EQ(data.y_axis().label(), ""); + EXPECT_EQ(data.title(), ""); + EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.x_axis().max(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().max(), 0.0f, tol); +} +//Check that plotting by lane gives you accurate values - valid entry test TEST(plot_logic, pf_clusters_by_lane) { const float tol = 1e-3f; @@ -127,10 +119,14 @@ TEST(plot_logic, pf_clusters_by_lane) metrics.set_naming_method(constants::FourDigit); metrics.legacy_channel_update(constants::HiSeq); - unittest::tile_metric_v2::create_metric_set(metrics.get_set()); + unittest::tile_metric_v2::create_expected(metrics.get()); model::plot::plot_data data; logic::plot::plot_by_lane(metrics, constants::ClusterCountPF, options, data); + + const double expected_val = 3.22777605; + EXPECT_NEAR(expected_val, data[0][0].y(), tol); + //data.size() refers to the number of series in the collection ASSERT_EQ(data.size(), 1u); EXPECT_EQ(data.title(), ""); EXPECT_EQ(data.x_axis().label(), "Lane"); @@ -138,10 +134,40 @@ TEST(plot_logic, pf_clusters_by_lane) EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); EXPECT_NEAR(data.x_axis().max(), 8.0f, tol); - EXPECT_NEAR(data.y_axis().max(), 3.9946798252130975, tol); + EXPECT_NEAR(data.y_axis().max(), 3.9946798252130975, tol); } +//Check that reading in no interop and plotting by lane prints out 0's and doesn't crash badly +TEST(plot_logic, pf_clusters_by_lane_empty_interop) +{ + const float tol = 1e-3f; + model::metrics::run_metrics metrics; + model::plot::filter_options options(constants::FourDigit); + std::vector reads(2); + reads[0] = model::run::read_info(1, 1, 26); + reads[1] = model::run::read_info(2, 27, 76); + metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), + std::vector(), model::run::image_dimensions(), reads)); + metrics.set_naming_method(constants::FourDigit); + metrics.legacy_channel_update(constants::HiSeq); + + model::plot::plot_data data; + logic::plot::plot_by_lane(metrics, constants::ClusterCountPF, options, data); + + //data.size() refers to the number of series, so this should still be 1 + ASSERT_EQ(data.size(), 0u); + EXPECT_EQ(data.title(), ""); + + EXPECT_EQ(data.x_axis().label(), ""); + EXPECT_EQ(data.y_axis().label(), ""); + EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.x_axis().max(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().max(), 0.0f, tol); +} + +//Checks that plotting q_score histogram works correctly - checks values TEST(plot_logic, q_score_histogram) { const float tol = 1e-3f; @@ -150,24 +176,32 @@ TEST(plot_logic, q_score_histogram) std::vector reads(2); reads[0] = model::run::read_info(1, 1, 26); reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); + metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), + std::vector(), model::run::image_dimensions(), reads)); metrics.legacy_channel_update(constants::HiSeq); metrics.set_naming_method(constants::FourDigit); - unittest::q_metric_v6::create_metric_set(metrics.get_set()); + unittest::q_metric_v6::create_expected(metrics.get()); metrics.finalize_after_load(); model::plot::plot_data data; logic::plot::plot_qscore_histogram(metrics, options, data); + const double expected_y[] = {0.721589, 0.217605, 0.005811, 8.61628}; + const size_t expected_x[] = {10, 20, 25, 30}; + size_t k = 0; + + //data.size() refers to the number of series in the collection ASSERT_EQ(data.size(), 1u); + for (size_t i = 0; i < data.size(); i++) + { + for (size_t j = 0; j < data[i].size(); j++) + { + EXPECT_NEAR(data[i][j].y(), static_cast(expected_y[k]), tol); + EXPECT_EQ(data[i][j].x(), expected_x[k]); + k++; + } + } + EXPECT_EQ(data.title(), "All Lanes"); EXPECT_EQ(data.x_axis().label(), "Q Score"); EXPECT_EQ(data.y_axis().label(), "Total (million)"); @@ -177,6 +211,37 @@ TEST(plot_logic, q_score_histogram) EXPECT_NEAR(data.y_axis().max(), 9.4780035018920898f, tol); } +//Not reading an interop in for Q-score Hist causes the title and labels to go away (only here, not in cycle/lane) +TEST(plot_logic, q_score_histogram_empty_interop) +{ + const float tol = 1e-3f; + model::metrics::run_metrics metrics; + model::plot::filter_options options(constants::FourDigit); + std::vector reads(2); + reads[0] = model::run::read_info(1, 1, 26); + reads[1] = model::run::read_info(2, 27, 76); + metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), + std::vector(), model::run::image_dimensions(), reads)); + metrics.legacy_channel_update(constants::HiSeq); + metrics.set_naming_method(constants::FourDigit); + + metrics.finalize_after_load(); + + model::plot::plot_data data; + logic::plot::plot_qscore_histogram(metrics, options, data); + + //data.size() refers to the number of series in the collection + ASSERT_EQ(data.size(), 0u); + EXPECT_EQ(data.title(), ""); + EXPECT_EQ(data.x_axis().label(), ""); + EXPECT_EQ(data.y_axis().label(), ""); + EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.x_axis().max(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().max(), 0.0f, tol); +} + +//Checks that q-score heatmap works as intended TEST(plot_logic, q_score_heatmap) { const float tol = 1e-3f; @@ -185,19 +250,12 @@ TEST(plot_logic, q_score_heatmap) std::vector reads(2); reads[0] = model::run::read_info(1, 1, 26); reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); + metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), + std::vector(), model::run::image_dimensions(), reads)); metrics.legacy_channel_update(constants::HiSeq); metrics.set_naming_method(constants::FourDigit); - unittest::q_metric_v6::create_metric_set(metrics.get_set()); + unittest::q_metric_v6::create_expected(metrics.get()); metrics.finalize_after_load(); model::plot::heatmap_data data; @@ -212,19 +270,25 @@ TEST(plot_logic, q_score_heatmap) EXPECT_NEAR(data.x_axis().max(), 3.0f, tol); EXPECT_NEAR(data.y_axis().max(), 40.0f, tol); double expected[] = - {0,0,0,0,0,0,0,0,0,9.1749,9.1749,9.1749,9.1749,9.1749,9.1749,9.1749,9.1749,9.1749,9.1749,4.06434,4.06434, - 4.06434,4.06434,4.06434,0.146682,0.146682,0.146682,0.146682,0.146682,95.7376,95.7376,95.7376,95.7376, - 95.7376,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8.26827,8.26827,8.26827,8.26827,8.26827,8.26827,8.26827,8.26827, - 8.26827,8.26827,1.53941,1.53941,1.53941,1.53941,1.53941,0.0376635,0.0376635,0.0376635,0.0376635,0.0376635, - 99.2799,99.2799,99.2799,99.2799,99.2799,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7.26372,7.26372,7.26372,7.26372, - 7.26372,7.26372,7.26372,7.26372,7.26372,7.26372,1.84695,1.84695,1.84695,1.84695,1.84695,0.0146203, - 0.0146203,0.0146203,0.0146203,0.0146203,100,100,100,100,100,0,0,0,0,0,0}; - - for(size_t row=0, k=0;row(expected[k]), data(row,col), 1e-3f); + EXPECT_NEAR(static_cast(expected[k]), data(row, col), 1e-3f); } } } @@ -278,7 +342,7 @@ TEST(plot_logic, q_score_heatmap_buffer) metrics.legacy_channel_update(constants::HiSeq); metrics.set_naming_method(constants::FourDigit); - unittest::q_metric_v6::create_metric_set(metrics.get_set()); + unittest::q_metric_v6::create_expected(metrics.get()); metrics.finalize_after_load(); model::plot::heatmap_data data1; @@ -286,24 +350,26 @@ TEST(plot_logic, q_score_heatmap_buffer) size_t row_count = logic::plot::count_rows_for_heatmap(metrics); size_t col_count = logic::plot::count_columns_for_heatmap(metrics); - std::vector buffer(row_count*col_count, 1); + std::vector buffer(row_count * col_count, 1); model::plot::heatmap_data data2; logic::plot::plot_qscore_heatmap(metrics, options, data2, &buffer.front()); - for(size_t row=0;row reads(2); reads[0] = model::run::read_info(1, 1, 26); @@ -312,7 +378,7 @@ TEST(plot_logic, flowcell_map) "", "", 1, - model::run::flowcell_layout(8, 2, 2, 36, 1, 1, std::vector(), constants::FourDigit), + model::run::flowcell_layout(num_lanes, 2, 2, 36, 1, 1, std::vector(), constants::FourDigit), std::vector(), model::run::image_dimensions(), reads @@ -320,22 +386,58 @@ TEST(plot_logic, flowcell_map) model::plot::filter_options options(constants::FourDigit, ALL_IDS, 0, constants::A, ALL_IDS, 1, 1); metrics.legacy_channel_update(constants::HiSeq); - unittest::extraction_metric_v2::create_metric_set(metrics.get_set()); - ASSERT_GT(metrics.extraction_metric_set().size(), 0u); + unittest::extraction_metric_v2::create_expected(metrics.get()); + ASSERT_GT(metrics.get< model::metrics::extraction_metric >().size(), 0u); model::plot::flowcell_data data; logic::plot::plot_flowcell_map(metrics, constants::Intensity, options, data); - ASSERT_EQ(data.row_count(), 8u); + std::vector actual_values; + actual_values.reserve(data.row_count() * data.column_count()); + for (size_t val = 0; val < data.row_count() * data.column_count(); val++) + { + actual_values.push_back(std::numeric_limits::quiet_NaN()); + } + float expected_vals[] = {302, 312, 349}; + std::pair indices[] = {std::make_pair(13, 6), std::make_pair(49, 6), + std::make_pair(85, 6)}; + size_t k = 0; + for (size_t i = 0; i < data.row_count(); i++) + { + for (size_t j = 0; j < data.column_count(); j++) + { + if (!std::isnan(data.at(i, j))) + { + actual_values[i * data.column_count() + j] = data.at(i, j); + EXPECT_NEAR(expected_vals[k], data.at(i, j), tol); + k++; + } + } + } + + size_t m = 0; + for (size_t val = 0; val < actual_values.size(); val++) + { + if (!std::isnan(actual_values[val])) + { + ASSERT_EQ(val / data.column_count(), indices[m].second); + ASSERT_EQ(val % data.column_count(), indices[m].first); + m++; + } + } + + ASSERT_EQ(data.row_count(), num_lanes); EXPECT_EQ(data.title(), "Intensity"); EXPECT_EQ(data.saxis().label(), "Intensity"); EXPECT_NEAR(data.saxis().min(), 302.0f, tol); EXPECT_NEAR(data.saxis().max(), 349.0f, tol); } -TEST(plot_logic, sample_qc) +//Checks that plot_flowcell_map with no interop read sets axis values cleanly (no crash) +TEST(plot_logic, flowcell_map_empty_interop) { const model::plot::filter_options::id_t ALL_IDS = model::plot::filter_options::ALL_IDS; const float tol = 1e-3f; + const size_t num_lanes = 8; model::metrics::run_metrics metrics; std::vector reads(2); reads[0] = model::run::read_info(1, 1, 26); @@ -344,7 +446,7 @@ TEST(plot_logic, sample_qc) "", "", 1, - model::run::flowcell_layout(8, 2, 2, 36, 1, 1, std::vector(), constants::FourDigit), + model::run::flowcell_layout(num_lanes, 2, 2, 36, 1, 1, std::vector(), constants::FourDigit), std::vector(), model::run::image_dimensions(), reads @@ -352,19 +454,196 @@ TEST(plot_logic, sample_qc) model::plot::filter_options options(constants::FourDigit, ALL_IDS, 0, constants::A, ALL_IDS, 1, 1); metrics.legacy_channel_update(constants::HiSeq); - unittest::index_metric_v1::create_metric_set(metrics.get_set()); - unittest::tile_metric_v2::create_metric_set(metrics.get_set()); - ASSERT_GT(metrics.index_metric_set().size(), 0u); + model::plot::flowcell_data data; + logic::plot::plot_flowcell_map(metrics, constants::Intensity, options, data); + ASSERT_EQ(data.row_count(), 0u); + ASSERT_EQ(data.column_count(), 0u); + EXPECT_EQ(data.title(), ""); + EXPECT_EQ(data.saxis().label(), ""); + EXPECT_NEAR(data.saxis().min(), 0.0f, tol); + EXPECT_NEAR(data.saxis().max(), 0.0f, tol); +} + +//Test to ensure that plot_sample_qc works as intended +TEST(plot_logic, sample_qc) +{ + const float tol = 1e-3f; + model::metrics::run_metrics metrics; + std::vector reads(2); + reads[0] = model::run::read_info(1, 1, 26); + reads[1] = model::run::read_info(2, 27, 76); + metrics.run_info( + model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 36, 1, 1, std::vector(), + constants::FourDigit), + std::vector(), model::run::image_dimensions(), reads)); + metrics.legacy_channel_update(constants::HiSeq); + + unittest::index_metric_v1::create_expected(metrics.get()); + unittest::tile_metric_v2::create_expected(metrics.get()); + ASSERT_GT(metrics.get< model::metrics::index_metric >().size(), 0u); model::plot::plot_data data; - logic::plot::plot_sample_qc(metrics, 1, data); + logic::plot::plot_sample_qc(metrics, 7, data); + ASSERT_EQ(data.size(), 1u); + ASSERT_EQ(data[0].size(), 3u); + ASSERT_EQ(data[0].color(), "Green"); EXPECT_EQ(data[0].title(), "% reads"); + EXPECT_NEAR(data[0][0].x(), 1, tol); + EXPECT_NEAR(data[0][0].y(), 0.04597f, tol); + EXPECT_NEAR(data[0][1].x(), 2, tol); + EXPECT_NEAR(data[0][1].y(), 0.04693f, tol); + EXPECT_NEAR(data[0][2].x(), 3, tol); + EXPECT_NEAR(data[0][2].y(), 0.04701f, tol); EXPECT_EQ(data.x_axis().label(), "Index Number"); EXPECT_EQ(data.y_axis().label(), "% Reads Identified (PF)"); EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); - EXPECT_NEAR(data.x_axis().max(), 1, tol); + EXPECT_NEAR(data.x_axis().max(), 4, tol); EXPECT_NEAR(data.y_axis().max(), 5, tol); } +//Test to make sure that plot_sample_qc stores 0's for axis values with no interop being read +TEST(plot_logic, sample_qc_empty_interop) +{ + const float tol = 1e-3f; + model::metrics::run_metrics metrics; + std::vector reads(2); + reads[0] = model::run::read_info(1, 1, 26); + reads[1] = model::run::read_info(2, 27, 76); + metrics.run_info( + model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 36, 1, 1, std::vector(), + constants::FourDigit), + std::vector(), model::run::image_dimensions(), reads)); + metrics.legacy_channel_update(constants::HiSeq); + + model::plot::plot_data data; + logic::plot::plot_sample_qc(metrics, 1, data); + ASSERT_EQ(data.size(), 0u); + EXPECT_EQ(data.x_axis().label(), ""); + EXPECT_EQ(data.y_axis().label(), ""); + EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); + EXPECT_NEAR(data.x_axis().max(), 0.0f, tol); + EXPECT_NEAR(data.y_axis().max(), 0.0f, tol); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Lists +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +//Test that all metrics printed out by list_by_cycle_metrics are actually cycle metrics +TEST(plot_logic, check_plot_by_cycle_list) +{ + std::vector types; + logic::plot::list_by_cycle_metrics(types); + for (size_t i = 0; i < types.size(); ++i) + { + EXPECT_FALSE(logic::utils::is_read_metric(types[i])); + EXPECT_FALSE(logic::utils::is_tile_metric(types[i])) + << types[i] << " " << constants::to_string(types[i].value()); + EXPECT_TRUE(logic::utils::is_cycle_metric(types[i])) + << types[i] << " " << constants::to_string(types[i].value()); + } +} + +//Test that all cycle metrics are actually printed out by list_by_cycle_metrics +TEST(plot_logic, check_all_cycle_list) +{ + //Test all cycle metrics are actually printed out by list_by_cycle_metrics + std::vector metrics; + std::vector cycle_metrics; + std::vector types; + constants::list_enums(metrics); + + for (size_t i = 0; i < metrics.size(); i++) + { + if ((logic::utils::is_cycle_metric(metrics[i]) && + std::find(cycle_metrics.begin(), cycle_metrics.end(), metrics[i]) == cycle_metrics.end())) + { + cycle_metrics.push_back(metrics[i]); + } + } + + logic::plot::list_by_cycle_metrics(types); + EXPECT_TRUE(types.size() == cycle_metrics.size()); +} + +//Test that all metrics printed out by list_by_lane_metrics are actually lane metrics +TEST(plot_logic, check_plot_by_lane_list) +{ + std::vector types; + logic::plot::list_by_lane_metrics(types); + for (size_t i = 0; i < types.size(); ++i) + { + EXPECT_TRUE(logic::utils::is_read_metric(types[i]) || logic::utils::is_tile_metric(types[i])); + EXPECT_FALSE(logic::utils::is_cycle_metric(types[i])); + } +} + +//Test that all lane metrics are actually printed out by list_by_lane_metrics +TEST(plot_logic, check_all_lane_list) +{ + //Test all lane (read / tile) metrics are actually printed out by list_by_lane_metrics + std::vector metrics; + std::vector lane_metrics; + std::vector types; + constants::list_enums(metrics); + + for (size_t i = 0; i < metrics.size(); i++) + { + if (((logic::utils::is_read_metric(metrics[i]) || logic::utils::is_tile_metric(metrics[i])) + && !logic::utils::is_cycle_metric(metrics[i])) && + std::find(lane_metrics.begin(), lane_metrics.end(), metrics[i]) == lane_metrics.end()) + { + lane_metrics.push_back(metrics[i]); + } + } + + logic::plot::list_by_lane_metrics(types); + EXPECT_TRUE(types.size() == lane_metrics.size()); +} + +//Test that all metrics printed out by list_flowcell_metrics are actually flowcell metrics +TEST(plot_logic, check_plot_flowcell_list) +{ + std::vector types; + logic::plot::list_flowcell_metrics(types); + for (size_t i = 0; i < types.size(); ++i) + { + EXPECT_TRUE(logic::utils::is_read_metric(types[i]) + || logic::utils::is_tile_metric(types[i]) + || logic::utils::is_cycle_metric(types[i])); + } +} + +//Test all flowcell metrics are actually printed out by list_flowcell_metrics +TEST(plot_logic, check_all_flowcell_list) +{ + std::vector metrics; + std::vector flowcell_metrics; + std::vector types; + constants::list_enums(metrics); + + for (size_t i = 0; i < metrics.size(); i++) + { + if (logic::utils::is_read_metric(metrics[i]) || + logic::utils::is_tile_metric(metrics[i]) || + logic::utils::is_cycle_metric(metrics[i])) + { + { + if (std::find(flowcell_metrics.begin(), flowcell_metrics.end(), metrics[i]) == flowcell_metrics.end()) + { + flowcell_metrics.push_back(metrics[i]); + } + } + } + } + + logic::plot::list_flowcell_metrics(types); + EXPECT_TRUE(types.size() == flowcell_metrics.size()); +} + + + + diff --git a/src/tests/interop/logic/summary_metrics_test.cpp b/src/tests/interop/logic/summary_metrics_test.cpp index a3b49a081..99268e48a 100644 --- a/src/tests/interop/logic/summary_metrics_test.cpp +++ b/src/tests/interop/logic/summary_metrics_test.cpp @@ -11,120 +11,95 @@ #include #include "interop/util/math.h" #include "interop/logic/summary/run_summary.h" -#include "interop/logic/summary/index_summary.h" +#include "src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h" #include "src/tests/interop/metrics/inc/error_metrics_test.h" #include "src/tests/interop/metrics/inc/extraction_metrics_test.h" #include "src/tests/interop/metrics/inc/tile_metrics_test.h" #include "src/tests/interop/metrics/inc/q_metrics_test.h" -#include "src/tests/interop/metrics/inc/index_metrics_test.h" #include "src/tests/interop/inc/abstract_regression_test_generator.h" +using namespace illumina::interop::model::summary; using namespace illumina::interop::model::metrics; using namespace illumina::interop::io; using namespace illumina::interop; using namespace illumina::interop::unittest; -/** Generate the actual metric set by reading in from hardcoded binary buffer +/** Setup for tests that compare two run summaries */ +struct run_summary_tests : public generic_test_fixture {}; + + +/** Check if two stats are nearly the same. If both are NaN, then this check succeeds * - * The expected metric set is provided by the generator. + * @todo Use this everywhere + * + * @param expected expected stat + * @param actual actual stat + * @param tol tolerance + * @return true if both numbers hold the same value, or their difference is less than tolerance */ -template -class run_summary_generator : public abstract_generator< model::summary::run_summary > +::testing::AssertionResult AreStatsNear(const metric_stat& expected, const metric_stat& actual, const float tol) { - typedef typename Gen::metric_set_t metric_set_t; -public: - /** Generate the expected and actual metric sets - * - * @param expected expected metric set - * @param actual actual metric set - */ - bool generate(model::summary::run_summary& expected, model::summary::run_summary& actual)const + ::testing::Message msg; + bool test_failed = false; + if(!std::isnan(expected.mean()) || !std::isnan(actual.mean())) { - const size_t lane_count = 8; - const size_t surface_count = 2; - const size_t swath_count = 4; - const size_t tile_count = 99; - const size_t sections_per_lane = 6; - const size_t lanes_per_section = 6; - Gen::create_summary(expected); - std::vector reads; - expected.copy_reads(reads); - actual = model::summary::run_summary(reads.begin(), reads.end(), lane_count, surface_count); - std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count, - surface_count, - swath_count, - tile_count, - sections_per_lane, - lanes_per_section), - channels, - model::run::image_dimensions(), - reads); - run_info.set_naming_method(constants::FourDigit); - model::metrics::run_metrics metrics(run_info); - Gen::create_metric_set(metrics.get_set()); - metrics.finalize_after_load(); - logic::summary::summarize_run_metrics(metrics, actual); - return true; + if(std::abs(expected.mean()-actual.mean()) >= tol) + { + msg << "Mean Expected: " << expected.mean() << " == Actual: " << actual.mean(); + test_failed=true; + } } - /** Create a copy of this object - * - * @return pointer to an abstract_generator - */ - abstract_generator< model::summary::run_summary >* clone()const + if(!std::isnan(expected.stddev()) || !std::isnan(actual.stddev())) { - return new run_summary_generator; + if(std::abs(expected.stddev()-actual.stddev()) >= tol) + { + if(test_failed) msg << " | "; + msg << "StdDev Expected: " << expected.stddev() << " == Actual: " << actual.stddev(); + test_failed=true; + } } - /** Write generator info to output stream - * - * @param out output stream - */ - void write(std::ostream& out)const + if(!std::isnan(expected.median()) || !std::isnan(actual.median())) { - out << "run_summary_generator<"<< Gen::name() << ">"; + if(std::abs(expected.median()-actual.median()) >= tol) + { + if(test_failed) msg << " | "; + msg << "Median Expected: " << expected.median() << " == Actual: " << actual.median(); + test_failed=true; + } } -}; - -/** Setup for tests that compare two run summaries */ -struct run_summary_tests : public generic_test_fixture< model::summary::run_summary > {}; - -run_summary_tests::generator_type run_summary_unit_test_generators[] = { - new run_summary_generator< error_metric_v3 >(), - new run_summary_generator< extraction_metric_v2 >(), - new run_summary_generator< q_metric_v4 >(), - new run_summary_generator< q_metric_v5 >(), - new run_summary_generator< q_metric_v6 >(), - new run_summary_generator< tile_metric_v2 >() -}; - - -// Setup unit tests for run summary tests -INSTANTIATE_TEST_CASE_P(run_summary_unit_test, - run_summary_tests, - ::testing::ValuesIn(run_summary_unit_test_generators)); - -#define EXPECT_STAT_NEAR(ACTUAL, EXPECTED, TOL) \ - if(!std::isnan(ACTUAL.mean()) || !std::isnan(EXPECTED.mean())) \ - EXPECT_NEAR(ACTUAL.mean(), EXPECTED.mean(), TOL); \ - if(!std::isnan(ACTUAL.stddev()) || !std::isnan(EXPECTED.stddev())) \ - EXPECT_NEAR(ACTUAL.stddev(), EXPECTED.stddev(), TOL); \ - if(!std::isnan(ACTUAL.median()) || !std::isnan(EXPECTED.median())) \ - EXPECT_NEAR(ACTUAL.median(), EXPECTED.median(), TOL) + if(test_failed) return ::testing::AssertionFailure(msg << " Tol: " << tol); + return ::testing::AssertionSuccess(); +} -#define EXPECT_CYCLE_EQ(ACTUAL, EXPECTED) \ - EXPECT_EQ(ACTUAL.first_cycle(), EXPECTED.first_cycle()); \ - EXPECT_EQ(ACTUAL.last_cycle(), EXPECTED.last_cycle()) +/** Check if two cycle states are the same. + * + * @todo Use this everywhere + * + * @param expected expected cycle state + * @param actual actual cycle state + * @return true if both are equal + */ +::testing::AssertionResult AreCycleStatesNear(const model::run::cycle_range& expected, + const model::run::cycle_range& actual) +{ + ::testing::Message msg; + bool test_failed = false; + if(expected.first_cycle() != actual.first_cycle()) + { + msg << "First Cycle Expected: " << expected.first_cycle() << " == Actual: " << actual.first_cycle(); + test_failed = true; + } + if(expected.last_cycle() != actual.last_cycle()) + { + msg << "Last Cycle Expected: " << expected.last_cycle() << " == Actual: " << actual.last_cycle(); + test_failed = true; + } + if(test_failed) return ::testing::AssertionFailure(msg); + return ::testing::AssertionSuccess(); +} -#define EXPECT_READ_EQ(ACTUAL, EXPECTED) \ - EXPECT_EQ(ACTUAL.first_cycle(), EXPECTED.first_cycle()); \ - EXPECT_EQ(ACTUAL.last_cycle(), EXPECTED.last_cycle()); \ - EXPECT_EQ(ACTUAL.number(), EXPECTED.number()); \ - EXPECT_EQ(ACTUAL.is_index(), EXPECTED.is_index()) +#define INTEROP_EXPECT_STAT_NEAR(EXPECTED, ACTUAL, TOL) EXPECT_TRUE(AreStatsNear(EXPECTED, ACTUAL, TOL)) +#define INTEROP_EXPECT_CYCLE_EQ(ACTUAL, EXPECTED) EXPECT_TRUE(AreCycleStatesNear(EXPECTED, ACTUAL)) /** Test if calculated run summary matches actual run summary * @@ -132,60 +107,82 @@ INSTANTIATE_TEST_CASE_P(run_summary_unit_test, */ TEST_P(run_summary_tests, run_summary) { - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); const float tol = 1e-7f; - EXPECT_EQ(actual.size(), expected.size()); - EXPECT_EQ(actual.lane_count(), expected.lane_count()); - EXPECT_EQ(actual.surface_count(), expected.surface_count()); - EXPECT_NEAR(actual.total_summary().error_rate(), expected.total_summary().error_rate(), tol); - EXPECT_NEAR(actual.total_summary().percent_aligned(), expected.total_summary().percent_aligned(), tol); - EXPECT_NEAR(actual.total_summary().first_cycle_intensity(), expected.total_summary().first_cycle_intensity(), tol); - EXPECT_NEAR(actual.total_summary().percent_gt_q30(), expected.total_summary().percent_gt_q30(), tol); - EXPECT_NEAR(actual.total_summary().yield_g(), expected.total_summary().yield_g(), tol); - EXPECT_NEAR(actual.total_summary().projected_yield_g(), expected.total_summary().projected_yield_g(), tol); - - - EXPECT_NEAR(actual.nonindex_summary().error_rate(), expected.nonindex_summary().error_rate(), tol); - EXPECT_NEAR(actual.nonindex_summary().percent_aligned(), expected.nonindex_summary().percent_aligned(), tol); - EXPECT_NEAR(actual.nonindex_summary().first_cycle_intensity(), expected.nonindex_summary().first_cycle_intensity(), tol); - EXPECT_NEAR(actual.nonindex_summary().percent_gt_q30(), expected.nonindex_summary().percent_gt_q30(), tol); - EXPECT_NEAR(actual.nonindex_summary().yield_g(), expected.nonindex_summary().yield_g(), tol); - EXPECT_NEAR(actual.nonindex_summary().projected_yield_g(), expected.nonindex_summary().projected_yield_g(), tol); - - const model::summary::cycle_state_summary& actual_cycle_summary = actual.cycle_state(); - const model::summary::cycle_state_summary& expected_cycle_summary = expected.cycle_state(); - EXPECT_CYCLE_EQ(actual_cycle_summary.extracted_cycle_range(), expected_cycle_summary.extracted_cycle_range()); - EXPECT_CYCLE_EQ(actual_cycle_summary.called_cycle_range(), expected_cycle_summary.called_cycle_range()); - EXPECT_CYCLE_EQ(actual_cycle_summary.qscored_cycle_range(), expected_cycle_summary.qscored_cycle_range()); - EXPECT_CYCLE_EQ(actual_cycle_summary.error_cycle_range(), expected_cycle_summary.error_cycle_range()); + ASSERT_EQ(actual.size(), expected.size()); + ASSERT_EQ(actual.lane_count(), expected.lane_count()); + ASSERT_EQ(actual.surface_count(), expected.surface_count()); + EXPECT_EQ(actual.channel_count(), expected.channel_count()); + + + INTEROP_EXPECT_NEAR(actual.total_summary().error_rate(), expected.total_summary().error_rate(), tol); + INTEROP_EXPECT_NEAR(actual.total_summary().percent_aligned(), expected.total_summary().percent_aligned(), tol); + INTEROP_EXPECT_NEAR(actual.total_summary().first_cycle_intensity(), expected.total_summary().first_cycle_intensity(), tol); + INTEROP_EXPECT_NEAR(actual.total_summary().percent_gt_q30(), expected.total_summary().percent_gt_q30(), tol); + INTEROP_EXPECT_NEAR(actual.total_summary().yield_g(), expected.total_summary().yield_g(), tol); + INTEROP_EXPECT_NEAR(actual.total_summary().projected_yield_g(), expected.total_summary().projected_yield_g(), tol); + + + INTEROP_EXPECT_NEAR(actual.nonindex_summary().error_rate(), expected.nonindex_summary().error_rate(), tol); + INTEROP_EXPECT_NEAR(actual.nonindex_summary().percent_aligned(), expected.nonindex_summary().percent_aligned(), tol); + INTEROP_EXPECT_NEAR(actual.nonindex_summary().first_cycle_intensity(), expected.nonindex_summary().first_cycle_intensity(), + tol); + INTEROP_EXPECT_NEAR(actual.nonindex_summary().percent_gt_q30(), expected.nonindex_summary().percent_gt_q30(), tol); + INTEROP_EXPECT_NEAR(actual.nonindex_summary().yield_g(), expected.nonindex_summary().yield_g(), tol); + INTEROP_EXPECT_NEAR(actual.nonindex_summary().projected_yield_g(), expected.nonindex_summary().projected_yield_g(), tol); + + const model::summary::cycle_state_summary &actual_cycle_summary = actual.cycle_state(); + const model::summary::cycle_state_summary &expected_cycle_summary = expected.cycle_state(); + INTEROP_EXPECT_CYCLE_EQ(actual_cycle_summary.extracted_cycle_range(), expected_cycle_summary.extracted_cycle_range()); + INTEROP_EXPECT_CYCLE_EQ(actual_cycle_summary.called_cycle_range(), expected_cycle_summary.called_cycle_range()); + INTEROP_EXPECT_CYCLE_EQ(actual_cycle_summary.qscored_cycle_range(), expected_cycle_summary.qscored_cycle_range()); + INTEROP_EXPECT_CYCLE_EQ(actual_cycle_summary.error_cycle_range(), expected_cycle_summary.error_cycle_range()); + } /** Test if calculated read summary matches actual read summary */ TEST_P(run_summary_tests, read_summary) { - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); ASSERT_EQ(actual.size(), expected.size()); + ASSERT_EQ(actual.lane_count(), expected.lane_count()); + ASSERT_EQ(actual.surface_count(), expected.surface_count()); const float tol = 1e-7f; - for(size_t read=0;read& expected_error_metrics= - expected_run_metrics.get_set(); + model::metric_base::metric_set &expected_error_metrics = + expected_run_metrics.get(); model::metrics::run_metrics actual_run_metrics(run_info); - model::metric_base::metric_set& actual_error_metrics= - actual_run_metrics.get_set(); - typedef model::metrics::error_metric::uint_t uint_t; - for(uint_t cycle_number=0;cycle_number<36;++cycle_number) + model::metric_base::metric_set &actual_error_metrics = + actual_run_metrics.get(); + typedef model::metrics::error_metric::uint_t uint_t; + for (uint_t cycle_number = 0; cycle_number < 36; ++cycle_number) { expected_error_metrics.insert(error_metric(1, 1101, 1 + cycle_number, 3.0f)); actual_error_metrics.insert(error_metric(1, 1101, 1 + cycle_number, 3.0f)); } - for(uint_t cycle_number=0;cycle_number<34;++cycle_number) + for (uint_t cycle_number = 0; cycle_number < 34; ++cycle_number) { actual_error_metrics.insert(error_metric(1, 1102, 1 + cycle_number, 1.0f)); } @@ -359,150 +397,339 @@ TEST(summary_metrics_test, cycle_35_cycle_34_tile) ASSERT_EQ(actual[0].size(), 1u); const float tol = 1e-7f; - const model::summary::lane_summary& actual_lane_summary = actual[0][0]; - const model::summary::lane_summary& expected_lane_summary = expected[0][0]; + const model::summary::lane_summary &actual_lane_summary = actual[0][0]; + const model::summary::lane_summary &expected_lane_summary = expected[0][0]; model::summary::metric_stat expected_stat(3.0f, 0.0f, 3.0f); - EXPECT_STAT_NEAR(expected_lane_summary.error_rate_35(), expected_stat, tol); - EXPECT_NEAR(actual_lane_summary.error_rate_35().mean(), expected_lane_summary.error_rate_35().mean(), tol); - expected_stat=model::summary::metric_stat(); - EXPECT_STAT_NEAR(expected_lane_summary.error_rate_50(), expected_stat, tol); - EXPECT_NEAR(actual_lane_summary.error_rate_50().mean(), expected_lane_summary.error_rate_50().mean(), tol); + INTEROP_EXPECT_STAT_NEAR(expected_lane_summary.error_rate_35(), expected_stat, tol); + INTEROP_EXPECT_NEAR(actual_lane_summary.error_rate_35().mean(), expected_lane_summary.error_rate_35().mean(), tol); + expected_stat = model::summary::metric_stat(); + INTEROP_EXPECT_STAT_NEAR(expected_lane_summary.error_rate_50(), expected_stat, tol); + INTEROP_EXPECT_NEAR(actual_lane_summary.error_rate_50().mean(), expected_lane_summary.error_rate_50().mean(), tol); } -TEST(summary_metrics_test, empty_run_metrics) -{ - model::metrics::run_metrics metrics; - model::summary::run_summary summary; - logic::summary::summarize_run_metrics(metrics, summary); -} - -/** TODO take tile metrics and index metrics from the same run */ -TEST(index_summary_test, lane_summary) +TEST(summary_metrics_test, clear_run_metrics) // TODO Expand to catch everything: probably use a fixture and the methods above { + const float tol = 1e-9f; const size_t lane_count = 8; - std::vector reads(1, model::run::read_info(1, 1, 3, false)); + const size_t surface_count = 2; + const size_t swath_count = 4; + const size_t tile_count = 99; + const size_t sections_per_lane = 6; + const size_t lanes_per_section = 6; std::vector channels; channels.push_back("Red"); channels.push_back("Green"); + std::vector reads; + reads.push_back(model::run::read_info(1, 1, 36)); + reads.push_back(model::run::read_info(2, 37, 42)); + reads.push_back(model::run::read_info(3, 43, 80)); model::run::info run_info("XX", "", 1, - model::run::flowcell_layout(lane_count), + model::run::flowcell_layout(lane_count, + surface_count, + swath_count, + tile_count, + sections_per_lane, + lanes_per_section), channels, model::run::image_dimensions(), reads); + run_info.set_naming_method(constants::FourDigit); + + model::metrics::run_metrics full_metrics(run_info); + tile_metric_v2::create_expected(full_metrics.get()); + model::summary::run_summary summary; + logic::summary::summarize_run_metrics(full_metrics, summary); + INTEROP_EXPECT_NEAR(summary.total_summary().percent_aligned(), 2.5863409042358398f, tol); - model::summary::index_flowcell_summary expected; - index_metric_v1::create_summary(expected); - model::summary::index_flowcell_summary actual; - model::metrics::run_metrics metrics(run_info); - try + + model::metrics::run_metrics partial_metrics(run_info); + extraction_metric_v2::create_expected(partial_metrics.get()); + logic::summary::summarize_run_metrics(partial_metrics, summary); + INTEROP_EXPECT_NEAR(summary.total_summary().percent_aligned(), 0, tol); + + model::metrics::run_metrics empty_metrics; + logic::summary::summarize_run_metrics(empty_metrics, summary); + INTEROP_EXPECT_NEAR(summary.total_summary().percent_aligned(), 0, tol); + EXPECT_EQ(summary.size(), 0u); +} + +TEST(summary_metrics_test, empty_run_metrics) +{ + const float tol = 1e-9f; + model::metrics::run_metrics metrics; + model::summary::run_summary summary; + logic::summary::summarize_run_metrics(metrics, summary); + INTEROP_EXPECT_NEAR(summary.total_summary().percent_aligned(), 0, tol); + EXPECT_EQ(summary.size(), 0u); +} + + + +//--------------------------------------------------------------------------------------------------------------------- +// Unit test section +//--------------------------------------------------------------------------------------------------------------------- + +/** Run the summary logic */ +struct summary_logic +{ + /** Run the summary logic + * + * @param metrics + * @param summary + */ + void operator()(model::metrics::run_metrics& metrics, + model::summary::run_summary& summary) + { + logic::summary::summarize_run_metrics(metrics, summary); + } + /** Get name of the logic + * + * @return name of the logic + */ + static const char* name() { - std::string data; - index_metric_v1::create_binary_data(data); - io::read_interop_from_string(data, - metrics.get_set()); - tile_metric_v2::create_binary_data(data); - io::read_interop_from_string(data, - metrics.get_set()); + return "Summary"; } - catch (const std::exception &) { } - logic::summary::summarize_index_metrics(metrics, actual); +}; - const float tol = 1e-7f; - ASSERT_EQ(expected.size(), actual.size()); - for(size_t lane=0;lane < expected.size();++lane) + +/** Generate the actual metric set by reading in from hardcoded binary buffer + * + * The expected metric set is provided by the generator. + */ +template +class run_summary_generator : public abstract_generator +{ + typedef typename Gen::metric_set_t metric_set_t; + typedef abstract_generator::parent_type base_t; +public: + /** Generate the expected and actual metric sets + * + * @param expected expected metric set + * @param actual actual metric set + */ + ::testing::AssertionResult generate(model::summary::run_summary &expected, + model::summary::run_summary &actual, + bool*) const { - const model::summary::index_lane_summary& expected_lane = expected[lane]; - const model::summary::index_lane_summary& actual_lane = actual[lane]; - EXPECT_EQ(expected_lane.total_reads(), actual_lane.total_reads()); - EXPECT_EQ(expected_lane.total_pf_reads(), actual_lane.total_pf_reads()); - EXPECT_NEAR(expected_lane.total_fraction_mapped_reads(), actual_lane.total_fraction_mapped_reads(), tol); - if(!std::isnan(expected_lane.mapped_reads_cv()) || !std::isnan(actual_lane.mapped_reads_cv())) - EXPECT_NEAR(expected_lane.mapped_reads_cv(), actual_lane.mapped_reads_cv(), tol); - EXPECT_NEAR(expected_lane.min_mapped_reads(), actual_lane.min_mapped_reads(), tol); - EXPECT_NEAR(expected_lane.max_mapped_reads(), actual_lane.max_mapped_reads(), tol); - ASSERT_EQ(expected_lane.size(), actual_lane.size()); - for(size_t index=0;index < expected_lane.size();++index) - { - const model::summary::index_count_summary& expected_count = expected_lane[index]; - const model::summary::index_count_summary& actual_count = actual_lane[index]; - EXPECT_EQ(expected_count.id(), actual_count.id()); - EXPECT_EQ(expected_count.index1(), actual_count.index1()); - EXPECT_EQ(expected_count.index2(), actual_count.index2()); - EXPECT_EQ(expected_count.count(), actual_count.count()); - EXPECT_EQ(expected_count.sample_id(), actual_count.sample_id()); - EXPECT_EQ(expected_count.project_name(), actual_count.project_name()); - EXPECT_NEAR(expected_count.fraction_mapped(), actual_count.fraction_mapped(), tol); - } + expected.clear(); + actual.clear(); + const size_t lane_count = 8; + const size_t surface_count = 2; + const size_t swath_count = 4; + const size_t tile_count = 99; + const size_t sections_per_lane = 6; + const size_t lanes_per_section = 6; + Gen::create_summary(expected); + std::vector reads; + expected.copy_reads(reads); + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + actual = model::summary::run_summary(reads.begin(), reads.end(), lane_count, surface_count, channels.size()); + model::run::info run_info("XX", + "", + 1, + model::run::flowcell_layout(lane_count, + surface_count, + swath_count, + tile_count, + sections_per_lane, + lanes_per_section), + channels, + model::run::image_dimensions(), + reads); + run_info.set_naming_method(constants::FourDigit); + model::metrics::run_metrics metrics(run_info); + Gen::create_expected(metrics.get()); + metrics.finalize_after_load(); + SummaryLogic summary_logic; + summary_logic(metrics, actual); + return ::testing::AssertionSuccess(); + } + + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + base_t clone() const + { + return new run_summary_generator; + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "run_summary_generator<" << Gen::name() << "_" << SummaryLogic::name() << ">"; } }; +/** Generator to test writing and reading an index summary */ +class summary_write_read_generator +{ +public: + /** Define parameter type */ + typedef int parameter_type; +public: + /** Constructor */ + summary_write_read_generator(const int){} + /** Generate the expected and actual metric sets + * + * @param expected expected plot data + * @param actual actual plot data + */ + ::testing::AssertionResult generate(model::summary::run_summary& expected, + model::summary::run_summary &actual) const + { + error_metric_v3::create_summary(expected); + extraction_metric_v2::create_summary(expected); + q_metric_v4::create_summary(expected); + tile_metric_v2::create_summary(expected); + corrected_intensity_metric_v2::create_summary(expected); + std::ostringstream oss; + oss << expected; + + std::istringstream iss(oss.str()); + iss >> actual; + return ::testing::AssertionSuccess(); + } + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "summary_write_read_generator"; + } +}; + +run_summary_tests::generator_type run_summary_unit_test_generators[] = { + new run_summary_generator(), + new run_summary_generator(), + new run_summary_generator(), + new run_summary_generator(), + new run_summary_generator(), + new run_summary_generator(), + new run_summary_generator(), + new run_summary_generator(), + + // Write/read + wrap(new standard_parameter_generator(0)) +}; + + +// Setup unit tests for run summary tests +INSTANTIATE_TEST_CASE_P(run_summary_unit_test, + run_summary_tests, + ::testing::ValuesIn(run_summary_unit_test_generators)); + + //--------------------------------------------------------------------------------------------------------------------- // Regression test section //--------------------------------------------------------------------------------------------------------------------- /** Generate the actual run summary writing out the expected and reading it back in again */ -class regression_test_summary_generator : public abstract_regression_test_generator< model::summary::run_summary > +template +class regression_test_summary_generator : public abstract_regression_test_generator { - typedef abstract_regression_test_generator< model::summary::run_summary > parent_t; + typedef abstract_regression_test_generator parent_t; public: - regression_test_summary_generator(const std::string& test_dir) : parent_t(test_dir){} - regression_test_summary_generator(const std::string& run_folder, const std::string& test_dir) : parent_t(run_folder, test_dir){} + /** Constructor + * + * @param test_dir sub folder where tests are stored + */ + regression_test_summary_generator(const std::string &test_dir) : parent_t(test_dir) + {} + + /** Constructor + * + * @param run_folder run folder with data + * @param test_dir sub folder where tests are stored + */ + regression_test_summary_generator(const std::string &run_folder, const std::string &test_dir) : parent_t(run_folder, + test_dir) + {} protected: /** Read the expected data from the baseline file into the model * * @param baseline_file baseline file * @param expected expected model data + * @return true if the file was found, and the read completed without failure */ - void read_expected(const std::string& baseline_file, model::summary::run_summary& expected)const + bool read_expected(const std::string &baseline_file, model::summary::run_summary &expected) const { - std::ifstream fin(baseline_file.c_str()); + std::ifstream fin(baseline_file.c_str(), std::ifstream::binary); + if( !fin.good() ) return false; fin >> expected; + if( fin.eof() ) return true; + return !fin.fail(); } + /** Read the actual data from the run folder * * @param run_folder run folder * @param actual actual model data + * @return true if data was generated */ - void generate_actual(const std::string& run_folder, model::summary::run_summary& actual)const + bool generate_actual(const std::string &run_folder, model::summary::run_summary &actual) const { model::metrics::run_metrics actual_metrics; actual_metrics.read(run_folder); - logic::summary::summarize_run_metrics(actual_metrics, actual); + if( actual_metrics.empty() ) return false; + Logic logic; + logic(actual_metrics, actual); + return actual.size() > 0; } + /** Write the actual data to the run folder * * @param baseline_file baseline file * @param actual actual model data */ - bool write_actual(const std::string& baseline_file, const model::summary::run_summary& actual)const + bool write_actual(const std::string &baseline_file, const model::summary::run_summary &actual) const { - std::ofstream fout(baseline_file.c_str()); + std::ofstream fout(baseline_file.c_str(), std::ofstream::binary); fout << actual; return fout.good(); } + /** Create a copy of the current object with the given run folder + * + * @param run_folder run folder + * @return pointer to new copy + */ + base_t clone(const std::string& run_folder)const + { + return new regression_test_summary_generator(run_folder, m_test_dir); + } + /** Create a copy of the current object * * @return pointer to new copy */ - base_type clone()const + base_t clone() const { - return new regression_test_summary_generator(m_run_folder, m_test_dir); + return new regression_test_summary_generator(*this); } + /** Write generator info to output stream * * @param out output stream */ - void write(std::ostream& out)const + void write(std::ostream &out) const { out << "regression_test_summary_generator - " << io::basename(m_run_folder); } }; -regression_test_summary_generator run_summary_regression_gen("summary"); +regression_test_summary_generator run_summary_regression_gen("summary"); + INSTANTIATE_TEST_CASE_P(run_summary_regression_test, run_summary_tests, ProxyValuesIn(run_summary_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/base_metric_tests.cpp b/src/tests/interop/metrics/base_metric_tests.cpp index 792b0f8e7..5f3fef79b 100644 --- a/src/tests/interop/metrics/base_metric_tests.cpp +++ b/src/tests/interop/metrics/base_metric_tests.cpp @@ -13,7 +13,7 @@ #ifdef _MSC_VER #pragma warning(push) - #pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants +#pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants #endif using namespace illumina::interop::model; @@ -21,63 +21,72 @@ using namespace illumina::interop::model::metric_base; TEST(base_metric_test, lane_from_id) { - base_metric::id_t id = base_metric::create_id(8, 1323); + const base_metric::id_t id = base_metric::create_id(8, 1323); EXPECT_EQ(base_metric::lane_from_id(id), 8u); - base_metric::id_t id2 = base_metric::create_id(1, 1); + const base_metric::id_t id2 = base_metric::create_id(1, 1); EXPECT_EQ(base_metric::lane_from_id(id2), 1u); } + TEST(base_metric_test, tile_from_id) { - base_metric::id_t id = base_metric::create_id(8, 1323); + const base_metric::id_t id = base_metric::create_id(8, 1323); EXPECT_EQ(base_metric::tile_from_id(id), 1323u); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// TEST(base_cycle_metric_test, lane_from_id) { - base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 2); + const base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 2); EXPECT_EQ(base_cycle_metric::lane_from_id(id), 8u); - base_metric::id_t id2 = base_cycle_metric::create_id(1, 1, 2); + const base_metric::id_t id2 = base_cycle_metric::create_id(1, 1, 2); EXPECT_EQ(base_cycle_metric::lane_from_id(id2), 1u); } + TEST(base_cycle_metric_test, tile_from_id) { - base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 2); + const base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 2); EXPECT_EQ(base_cycle_metric::tile_from_id(id), 1323u); + EXPECT_EQ(base_cycle_metric::reserved_from_id(id), 0u); } + TEST(base_cycle_metric_test, cycle_from_id) { - base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 10); + const base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 10); EXPECT_EQ(base_cycle_metric::cycle_from_id(id), 10u); + EXPECT_EQ(base_cycle_metric::reserved_from_id(id), 0u); } TEST(base_cycle_metric_test, tile_hash_from_id) { - base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 10); + const base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 10); EXPECT_EQ(base_cycle_metric::tile_hash_from_id(id), base_metric::create_id(8, 1323)); + EXPECT_EQ(base_cycle_metric::reserved_from_id(id), 0u); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// TEST(base_read_metric_test, lane_from_id) { - base_metric::id_t id = base_read_metric::create_id(8, 1323, 2); + const base_metric::id_t id = base_read_metric::create_id(8, 1323, 2); EXPECT_EQ(base_read_metric::lane_from_id(id), 8u); - base_metric::id_t id2 = base_read_metric::create_id(1, 1, 2); + const base_metric::id_t id2 = base_read_metric::create_id(1, 1, 2); EXPECT_EQ(base_read_metric::lane_from_id(id2), 1u); } + TEST(base_read_metric_test, tile_from_id) { - base_metric::id_t id = base_read_metric::create_id(8, 1323, 2); + const base_metric::id_t id = base_read_metric::create_id(8, 1323, 2); EXPECT_EQ(base_read_metric::tile_from_id(id), 1323u); } + TEST(base_read_metric_test, read_from_id) { - base_metric::id_t id = base_read_metric::create_id(8, 1323, 3); + const base_metric::id_t id = base_read_metric::create_id(8, 1323, 3); EXPECT_EQ(base_read_metric::read_from_id(id), 3u); } + TEST(base_read_metric_test, tile_hash_from_id) { - base_metric::id_t id = base_read_metric::create_id(8, 1323, 10); + const base_metric::id_t id = base_read_metric::create_id(8, 1323, 10); EXPECT_EQ(base_read_metric::tile_hash_from_id(id), base_metric::create_id(8, 1323)); } diff --git a/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp b/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp index 3224f4144..fca38f3fd 100644 --- a/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp +++ b/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp @@ -67,7 +67,8 @@ INSTANTIATE_TEST_CASE_P(corrected_intensity_metric_unit_test, TEST_P(corrected_intensity_metrics_tests, test_metric_io_fidelity) { typedef corrected_intensity_metric_set::const_iterator const_iterator; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); const float tol = 1e-3f; EXPECT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); @@ -87,8 +88,7 @@ TEST_P(corrected_intensity_metrics_tests, test_metric_io_fidelity) } if(expected.version() == 2) { - if (!std::isnan(it_expected->signal_to_noise()) || !std::isnan(it_actual->signal_to_noise())) - EXPECT_NEAR(it_expected->signal_to_noise(), it_actual->signal_to_noise(), tol); + INTEROP_ASSERT_NEAR(it_expected->signal_to_noise(), it_actual->signal_to_noise(), tol); } for(ptrdiff_t i=-1;icalled_counts(static_cast(i)), @@ -120,7 +120,7 @@ TEST(corrected_intensity_metrics_test, test_percent_base_nan) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Use the sub folder `metrics` in the baseline folder to contain all the baseline data -regression_test_metric_generator corrected_intensity_regression_gen("metrics"); +regression_test_metric_generator corrected_intensity_regression_gen; /** * This method populates the regression test with run folders specified on the command line. The run folders diff --git a/src/tests/interop/metrics/coverage_test.cpp b/src/tests/interop/metrics/coverage_test.cpp new file mode 100644 index 000000000..47bd2047f --- /dev/null +++ b/src/tests/interop/metrics/coverage_test.cpp @@ -0,0 +1,105 @@ +/** Unit tests for test coverage over metrics and their formats + * + * + * @file + * @date 11/20/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include +#include "interop/io/metric_stream.h" +#include "src/tests/interop/metrics/inc/metric_generator.h" +#include "src/tests/interop/metrics/inc/metric_format_fixtures.h" +#include "src/tests/interop/metrics/inc/format_registry.h" + +#ifdef _MSC_VER +#pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants +#endif + + +using namespace illumina::interop; +using namespace illumina::interop::unittest; + + + +format_registry actual_format_registry; + +/** Fixture to track formats available and being tested */ +template +struct metric_coverage_test : public ::testing::Test, public TestSetup +{ + /** Type of metric */ + typedef typename TestSetup::metric_t metric_t; + /** Constructor */ + metric_coverage_test() + { + actual_format_registry(metric_t(), TestSetup::VERSION); + } +}; + +TYPED_TEST_CASE_P(metric_coverage_test); + + +TYPED_TEST_P(metric_coverage_test, sentinel){} + +REGISTER_TYPED_TEST_CASE_P(metric_coverage_test, + sentinel +); + + +INSTANTIATE_TYPED_TEST_CASE_P(Public, metric_coverage_test, PublicFormats); + +/** Confirm that every metric and every version of their format is being tested here */ +TEST(metric_coverage_test, stream_tests) +{ + model::metrics::run_metrics run; + format_registry expected_format_registry; + run.metrics_callback(expected_format_registry); + std::sort(actual_format_registry.m_keys.begin(), actual_format_registry.m_keys.end()); + std::sort(expected_format_registry.m_keys.begin(), expected_format_registry.m_keys.end()); + ASSERT_THAT(actual_format_registry.m_keys, ::testing::ContainerEq(expected_format_registry.m_keys)); + for(format_registry::const_key_iterator it = expected_format_registry.m_keys.begin(); + it != expected_format_registry.m_keys.end();++it) + { + EXPECT_THAT(actual_format_registry.m_format_map[*it], + ::testing::ContainerEq(expected_format_registry.m_format_map[*it])) << *it; + } +} + +/** Fixture to track formats available and being tested */ +template +struct format_coverage_test : public ::testing::Test +{ + /** Constructor */ + format_coverage_test() : actual(TestSetup::instance()) + { + model::metrics::run_metrics run; + run.metrics_callback(expected); + } + /** Expected format registry (from run metrics) */ + format_registry expected; + /** Actual format registry (from tests) */ + format_registry& actual; +}; +typedef ::testing::Types< + hardcoded_metric_registry_t, + write_read_metric_registry_t +> generators_t; +TYPED_TEST_CASE(format_coverage_test, generators_t); + +/** Confirm that every metric and every version of their format is being tested here */ +TYPED_TEST(format_coverage_test, format) +{ + std::sort(TestFixture::actual.m_keys.begin(), TestFixture::actual.m_keys.end()); + std::sort(TestFixture::expected.m_keys.begin(), TestFixture::expected.m_keys.end()); + ASSERT_THAT(TestFixture::actual.m_keys, ::testing::ContainerEq(TestFixture::expected.m_keys)); + for(format_registry::const_key_iterator it = TestFixture::expected.m_keys.begin(); + it != TestFixture::expected.m_keys.end();++it) + { + EXPECT_THAT(TestFixture::actual.m_format_map[*it], + ::testing::ContainerEq(TestFixture::expected.m_format_map[*it])) << *it; + } +} + + diff --git a/src/tests/interop/metrics/error_metrics_test.cpp b/src/tests/interop/metrics/error_metrics_test.cpp index 067cde7ca..d7fdb2f6f 100644 --- a/src/tests/interop/metrics/error_metrics_test.cpp +++ b/src/tests/interop/metrics/error_metrics_test.cpp @@ -44,7 +44,8 @@ INSTANTIATE_TEST_CASE_P(error_metric_unit_test, */ TEST_P(error_metrics_tests, compare_expected_actual) { - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); ASSERT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); EXPECT_EQ(actual.max_cycle(), expected.max_cycle()) << actual.offset_map().size() @@ -71,7 +72,7 @@ TEST_P(error_metrics_tests, compare_expected_actual) TEST(error_metrics_single_test, test_tile_metric_count_for_lane) { error_metric_set metrics; - error_metric_v3::create_metric_set(metrics); + error_metric_v3::create_expected(metrics); EXPECT_EQ(metrics.tile_numbers_for_lane(7).size(), 1u); } @@ -80,7 +81,7 @@ TEST(error_metrics_single_test, test_tile_metric_count_for_lane) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator error_regression_gen("metrics"); +regression_test_metric_generator error_regression_gen; INSTANTIATE_TEST_CASE_P(error_metric_regression_test, error_metrics_tests, ProxyValuesIn(error_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/extraction_metrics_test.cpp b/src/tests/interop/metrics/extraction_metrics_test.cpp index 6df4dee0f..d59dfb81a 100644 --- a/src/tests/interop/metrics/extraction_metrics_test.cpp +++ b/src/tests/interop/metrics/extraction_metrics_test.cpp @@ -45,7 +45,8 @@ INSTANTIATE_TEST_CASE_P(extraction_metric_unit_test, TEST_P(extraction_metrics_tests, test_read_write) { typedef extraction_metric_set::const_iterator const_iterator; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); const float tol = 1e-7f; EXPECT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); @@ -78,7 +79,7 @@ TEST_P(extraction_metrics_tests, test_read_write) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator extraction_regression_gen("metrics"); +regression_test_metric_generator extraction_regression_gen; INSTANTIATE_TEST_CASE_P(extraction_metric_regression_test, extraction_metrics_tests, ProxyValuesIn(extraction_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/image_metrics_test.cpp b/src/tests/interop/metrics/image_metrics_test.cpp index 41eca5cae..8385641f8 100644 --- a/src/tests/interop/metrics/image_metrics_test.cpp +++ b/src/tests/interop/metrics/image_metrics_test.cpp @@ -27,7 +27,7 @@ struct image_metrics_tests : public generic_test_fixture< image_metric_set > {}; image_metrics_tests::generator_type image_unit_test_generators[] = { - wrap(new hardcoded_metric_generator< image_metric_v1 >) , + wrap(new hardcoded_metric_generator< image_metric_v1 >), wrap(new write_read_metric_generator< image_metric_v1 >), wrap(new hardcoded_metric_generator< image_metric_v2 >) , wrap(new write_read_metric_generator< image_metric_v2 >) @@ -48,7 +48,8 @@ INSTANTIATE_TEST_CASE_P(image_metric_unit_test, TEST_P(image_metrics_tests, test_read_write) { typedef image_metric_set::const_iterator const_iterator; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); EXPECT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); ASSERT_EQ(actual.channel_count(), expected.channel_count()); @@ -74,7 +75,7 @@ TEST_P(image_metrics_tests, test_read_write) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator image_regression_gen("metrics"); +regression_test_metric_generator image_regression_gen; INSTANTIATE_TEST_CASE_P(image_metric_regression_test, image_metrics_tests, ProxyValuesIn(image_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h b/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h index 11c6c640e..99ac8040e 100644 --- a/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h +++ b/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h @@ -25,7 +25,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { typedef metric_t::uint_t uint_t; typedef metric_t::ushort_t ushort_t; @@ -34,7 +34,7 @@ namespace illumina{ namespace interop { namespace unittest const ushort_t corrected_int_called1[] = {4070, 4074, 4029, 3972}; const uint_t called_counts1[] = {0, 698433, 548189, 548712, 646638}; metrics.insert( - metric_t(1, 1104, 25, 1063, 11.9458876f, to_vector(corrected_int_called1), to_vector(corrected_int_all1), + metric_t(1, 1103, 25, 1063, 11.9458876f, to_vector(corrected_int_called1), to_vector(corrected_int_all1), to_vector(called_counts1))); const ushort_t corrected_int_all2[] = {1558, 1151, 1158, 1293}; const uint_t called_counts2[] = {10938, 733661, 537957, 543912, 615504}; @@ -58,7 +58,7 @@ namespace illumina{ namespace interop { namespace unittest { const int tmp[] = { - 2, 48, 1, 0, 80, 4, 25, 0, 39, 4, 189, 4, 198, 3, 192, 3, 71, 4, 230, 15, 234, 15, 189, 15, 132, + 2, 48, 1, 0, 79, 4, 25, 0, 39, 4, 189, 4, 198, 3, 192, 3, 71, 4, 230, 15, 234, 15, 189, 15, 132, 15, 0, 0, 0, 0, 65, 168, 10, 0, 93, 93, 8, 0, 104, 95, 8, 0, 238, 221, 9, 0, 91, 34, 63, 65, 1, 0, 80, 4, 1, 0, 15, 5, 22, 6, 127, 4, 134, 4, 13, 5, 149, 19, 119, 19, 51, 19, 68, 19, 186, 42, 0, 0, 221, 49, 11, 0, 101, 53, 8, 0, 168, 76, 8, 0, 80, 100, 9, 0, 5, 226, 84, 65, 1, 0, 81, 4, 25, 0, 1, @@ -72,13 +72,13 @@ namespace illumina{ namespace interop { namespace unittest */ static void create_summary(model::summary::run_summary& summary) { - const size_t lane_count = 1; - const size_t surface_count = 1; + const size_t lane_count = 0; + const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 27, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); - summary[0][0].cycle_state().called_cycle_range(model::run::cycle_range(1, 25)); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary.cycle_state().called_cycle_range(model::run::cycle_range(1, 25)); } }; @@ -95,7 +95,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { typedef metric_t::uint_t uint_t; typedef metric_t::ushort_t ushort_t; @@ -132,13 +132,13 @@ namespace illumina{ namespace interop { namespace unittest */ static void create_summary(model::summary::run_summary& summary) { - const size_t lane_count = 1; - const size_t surface_count = 1; + const size_t lane_count = 0; + const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 27, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); - summary[0][0].cycle_state().called_cycle_range(model::run::cycle_range(3, 3)); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary.cycle_state().called_cycle_range(model::run::cycle_range(3, 3)); } }; diff --git a/src/tests/interop/metrics/inc/error_metrics_test.h b/src/tests/interop/metrics/inc/error_metrics_test.h index bbd95bc7b..ac7471ee8 100644 --- a/src/tests/interop/metrics/inc/error_metrics_test.h +++ b/src/tests/interop/metrics/inc/error_metrics_test.h @@ -28,7 +28,7 @@ namespace illumina { namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t &metrics) + static void create_expected(metric_set_t &metrics) { metrics = metric_set_t(VERSION); metrics.insert(metric_t(7, 1114, 1, 0.450100899f)); @@ -61,10 +61,11 @@ namespace illumina { namespace interop { namespace unittest { const size_t lane_count = 1; const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[] = { model::run::read_info(1, 1, 3, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary[0][0].lane(7); summary[0][0].error_rate(model::summary::metric_stat(0.67515134811401367f, 0, 0.67515134811401367f)); summary[0][0][0].error_rate(model::summary::metric_stat(0.67515134811401367f, 0, 0.67515134811401367f)); diff --git a/src/tests/interop/metrics/inc/extraction_metrics_test.h b/src/tests/interop/metrics/inc/extraction_metrics_test.h index c9d831d48..24da4c652 100644 --- a/src/tests/interop/metrics/inc/extraction_metrics_test.h +++ b/src/tests/interop/metrics/inc/extraction_metrics_test.h @@ -27,7 +27,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { metrics = metric_set_t(VERSION); @@ -66,10 +66,11 @@ namespace illumina{ namespace interop { namespace unittest { const size_t lane_count = 1; const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 2, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary[0][0].lane(7); summary[0][0].first_cycle_intensity(model::summary::metric_stat(321, 24.75883674621582f, 312)); summary[0][0][0].tile_count(2); diff --git a/src/tests/interop/metrics/inc/format_registry.h b/src/tests/interop/metrics/inc/format_registry.h new file mode 100644 index 000000000..214a8c84d --- /dev/null +++ b/src/tests/interop/metrics/inc/format_registry.h @@ -0,0 +1,94 @@ +/** Register all metrics and formats being tested + * + * @file + * @date 11/20/16. + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include +#include +#include +#include "interop/constants/enums.h" +#include "interop/io/metric_stream.h" + +#ifdef _MSC_VER +#pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants +#endif + +namespace illumina{ namespace interop { namespace unittest +{ + /** Track every metrics and its available binary formats */ + class format_registry + { + typedef std::set version_set_t; + typedef std::vector key_vector_t; + typedef std::map format_map_t; + public: + /** Define an iterator to the keys identifying each metric */ + typedef key_vector_t::const_iterator const_key_iterator; + + public: + /** Insert a metric type and format version into the tracker + * + * Populate actual from tests + * + * @param version version of the format + */ + template + void operator()(const MetricType&, int version) + { + const std::string name = io::paths::interop_basename(); + if(m_format_map.find(name) == m_format_map.end()) + { + m_format_map[name] = version_set_t(); + m_keys.push_back(name); + } + m_format_map[name].insert(version); + } + /** Insert a metric set into the tracker + * + * Populate expected using run metrics + */ + template + void operator()(const MetricSet&) + { + typedef typename MetricSet::metric_type metric_t; + // Dyanmic phasing does not have a read or write format + // QByLane uses the same format as QMetrics + // Neither of these is tested + const constants::metric_group group = static_cast(MetricSet::TYPE); + if(group == constants::QByLane) + return; + const std::string name = io::paths::interop_basename(); + if(m_format_map.find(name) == m_format_map.end()) + { + m_format_map[name] = version_set_t(); + m_keys.push_back(name); + } + io::copy_versions(std::inserter(m_format_map[name], m_format_map[name].begin())); + } + /** Map each metric to a set of version formats */ + format_map_t m_format_map; + /** Map a list of metrics */ + key_vector_t m_keys; + }; + + /** Create a format register for a single argument template class */ + templateclass T> + struct registry_factory + { + /** Get a static instance of format registery + * + * @return format registry + */ + static format_registry& instance() + { + static format_registry _instance; + return _instance; + } + }; +}}} + + diff --git a/src/tests/interop/metrics/inc/image_metrics_test.h b/src/tests/interop/metrics/inc/image_metrics_test.h index 2ff3aa9c6..2ceb1ed72 100644 --- a/src/tests/interop/metrics/inc/image_metrics_test.h +++ b/src/tests/interop/metrics/inc/image_metrics_test.h @@ -30,7 +30,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { metrics = metric_set_t(VERSION); typedef metric_t::ushort_t ushort_t; @@ -83,7 +83,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { metrics = metric_set_t(header_t(2), VERSION); typedef metric_t::ushort_t ushort_t; diff --git a/src/tests/interop/metrics/inc/index_metrics_test.h b/src/tests/interop/metrics/inc/index_metrics_test.h index 8d87abf6e..9366c452c 100644 --- a/src/tests/interop/metrics/inc/index_metrics_test.h +++ b/src/tests/interop/metrics/inc/index_metrics_test.h @@ -34,7 +34,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { typedef metric_t::index_info_t index_info_t; metrics = metric_set_t(VERSION); @@ -92,6 +92,7 @@ namespace illumina{ namespace interop { namespace unittest index_summary[lane].push_back(model::summary::index_count_summary(3, "CAGATCCA", "AAGGTTCA", "3", "TSCAIndexes", 4578, 0.047f)); } } + } }; diff --git a/src/tests/interop/metrics/inc/metric_format_fixtures.h b/src/tests/interop/metrics/inc/metric_format_fixtures.h index 6ea80549e..d90ca2f7c 100644 --- a/src/tests/interop/metrics/inc/metric_format_fixtures.h +++ b/src/tests/interop/metrics/inc/metric_format_fixtures.h @@ -23,22 +23,25 @@ namespace illumina{ namespace interop { namespace unittest typedef ::testing::Types - < - corrected_intensity_metric_v2, - corrected_intensity_metric_v3, - error_metric_v3, - extraction_metric_v2, - image_metric_v1, - image_metric_v2, - index_metric_v1, - q_collapsed_metric_v2, - q_collapsed_metric_v6, - q_metric_v4, - q_metric_v5, - q_metric_v6, - q_metric_v6_unbinned, - tile_metric_v2 - > PublicFormats; + < + corrected_intensity_metric_v2, + corrected_intensity_metric_v3, + error_metric_v3, + extraction_metric_v2, + image_metric_v1, + image_metric_v2, + index_metric_v1, + q_collapsed_metric_v2, + q_collapsed_metric_v3, + q_collapsed_metric_v4, + q_collapsed_metric_v5, + q_collapsed_metric_v6, + q_metric_v4, + q_metric_v5, + q_metric_v6, + q_metric_v6_unbinned, + tile_metric_v2 + > PublicFormats; }}} diff --git a/src/tests/interop/metrics/inc/metric_generator.h b/src/tests/interop/metrics/inc/metric_generator.h index afd9440ff..abb8be92f 100644 --- a/src/tests/interop/metrics/inc/metric_generator.h +++ b/src/tests/interop/metrics/inc/metric_generator.h @@ -9,10 +9,18 @@ #include "interop/io/metric_file_stream.h" #include "src/tests/interop/inc/regression_test_data.h" #include "src/tests/interop/inc/abstract_regression_test_generator.h" +#include "src/tests/interop/metrics/inc/format_registry.h" namespace illumina{ namespace interop { namespace unittest { - + template + class hardcoded_metric_generator; + /** Hardcoded metric registry */ + typedef registry_factory hardcoded_metric_registry_t; + template + class write_read_metric_generator; + /** Write-read metric registry */ + typedef registry_factory write_read_metric_registry_t; /** Generate the actual metric set by reading in from hardcoded binary buffer * * The expected metric set is provided by the generator. @@ -21,21 +29,27 @@ namespace illumina{ namespace interop { namespace unittest class hardcoded_metric_generator : public abstract_generator< typename Gen::metric_set_t > { typedef typename Gen::metric_set_t metric_set_t; - typedef typename abstract_generator::base_type parent_t; + typedef typename Gen::metric_t metric_t; + typedef typename abstract_generator::parent_type parent_t; public: + /** Constructor */ + hardcoded_metric_generator() + { + hardcoded_metric_registry_t::instance()(metric_t(), Gen::VERSION); + } /** Generate the expected and actual metric sets * * @param expected expected metric set * @param actual actual metric set */ - bool generate(metric_set_t& expected, metric_set_t& actual)const + ::testing::AssertionResult generate(metric_set_t& expected, metric_set_t& actual, bool*)const { actual.clear(); - Gen::create_metric_set(expected); - std::vector< ::uint8_t > binary_data; + Gen::create_expected(expected); + std::string binary_data;//std::vector< ::uint8_t > binary_data; Gen::create_binary_data(binary_data); io::read_interop_from_string(binary_data, actual); - return true; + return ::testing::AssertionSuccess(); } /** Create a copy of this object * @@ -60,23 +74,28 @@ namespace illumina{ namespace interop { namespace unittest class write_read_metric_generator : public abstract_generator< typename Gen::metric_set_t > { typedef typename Gen::metric_set_t metric_set_t; - typedef typename abstract_generator::base_type parent_t; + typedef typename Gen::metric_t metric_t; + typedef typename abstract_generator::parent_type parent_t; public: /** Constructor */ - write_read_metric_generator() : abstract_generator< typename Gen::metric_set_t >(1){} + write_read_metric_generator() : abstract_generator< typename Gen::metric_set_t >(1) + { + write_read_metric_registry_t::instance()(metric_t(), Gen::VERSION); + } /** Generate the expected and actual metric sets * * @param expected expected metric set * @param actual actual metric set */ - bool generate(metric_set_t& expected, metric_set_t& actual)const + ::testing::AssertionResult generate(metric_set_t& expected, metric_set_t& actual, bool*)const { actual.clear(); - Gen::create_metric_set(expected); + Gen::create_expected(expected); std::ostringstream fout; illumina::interop::io::write_metrics(fout, expected); + //print_actual(std::cout, fout.str()); io::read_interop_from_string(fout.str(), actual); - return true; + return ::testing::AssertionSuccess(); } /** Create a copy of this object * @@ -91,7 +110,21 @@ namespace illumina{ namespace interop { namespace unittest { out << "write_read_metric_generator<" << Gen::name() << ">"; } + + private: + static void print_actual(std::ostream& out, const std::string& data) + { + for(size_t i=0;i { typedef abstract_regression_test_generator< MetricSet > parent_t; - typedef typename parent_t::base_type base_type; + typedef typename parent_t::base_t base_type; typedef MetricSet metric_set_t; + enum{INCOMPLETE=250, NOT_FOUND=251}; public: typedef typename abstract_regression_test_generator< MetricSet >::parent_type parent_type; public: /** Constructor - * - * @param test_dir subdirectory for baseline data */ - regression_test_metric_generator(const std::string& test_dir) : parent_t(test_dir, 2) + regression_test_metric_generator() : parent_t("InterOp", 2) { } /** Constructor * * @param run_folder run folder - * @param test_dir subdirectory for baseline data */ - regression_test_metric_generator(const std::string& run_folder, const std::string& test_dir) : - parent_t(run_folder, test_dir, 2) + regression_test_metric_generator(const std::string& run_folder) : + parent_t(run_folder, "InterOp", 2) { } @@ -127,8 +158,9 @@ namespace illumina{ namespace interop { namespace unittest * * @param baseline_file baseline file * @param expected expected model data + * @return true if the file was found, and the read completed without failure */ - void read_expected(const std::string& baseline_file, metric_set_t& expected)const + bool read_expected(const std::string& baseline_file, metric_set_t& expected)const { expected.clear(); try @@ -136,16 +168,17 @@ namespace illumina{ namespace interop { namespace unittest illumina::interop::io::read_interop(baseline_file, expected); } // Should never have an incomplete file in baseline, set sentinel to detect - catch(const io::incomplete_file_exception&){expected.set_version(250);} + catch(const io::incomplete_file_exception&){expected.set_version(INCOMPLETE);return true;} // Ensure missing file is expected - catch(const io::file_not_found_exception&){expected.set_version(251);} + catch(const io::file_not_found_exception&){expected.set_version(NOT_FOUND); return false;} + return true; } /** Read the actual data from the run folder * * @param run_folder run folder * @param actual actual model data */ - void generate_actual(const std::string& run_folder, metric_set_t& actual)const + bool generate_actual(const std::string& run_folder, metric_set_t& actual)const { actual.clear(); try @@ -153,9 +186,10 @@ namespace illumina{ namespace interop { namespace unittest illumina::interop::io::read_interop(run_folder, actual); } // Ensure missing file is expected - catch(const io::incomplete_file_exception&){if(actual.empty()) actual.set_version(251);} + catch(const io::incomplete_file_exception&){if(actual.empty()) actual.set_version(NOT_FOUND);} // Ensure file is missing - catch(const io::file_not_found_exception&){actual.set_version(251);} + catch(const io::file_not_found_exception&){actual.set_version(NOT_FOUND);} + return !actual.empty(); } /** Write the actual data to the run folder * @@ -164,17 +198,24 @@ namespace illumina{ namespace interop { namespace unittest */ bool write_actual(const std::string& baseline_file, const metric_set_t& actual)const { - io::mkdir(parent_t::baseline()); - io::mkdir(io::combine(parent_t::baseline(), "InterOp")); return illumina::interop::io::write_interop(baseline_file, actual); } + /** Create a copy of the current object with the given run folder + * + * @param run_folder run folder + * @return pointer to new copy + */ + base_type clone(const std::string& run_folder)const + { + return new regression_test_metric_generator(run_folder); + } /** Create a copy of the current object * * @return pointer to new copy */ base_type clone()const { - return new regression_test_metric_generator(parent_t::m_run_folder, parent_t::m_test_dir); + return new regression_test_metric_generator(*this); } /** Write generator info to output stream * @@ -185,6 +226,14 @@ namespace illumina{ namespace interop { namespace unittest out << "regression_test_metric_generator< "<< metric_set_t::prefix() << metric_set_t::suffix() << "> - " << io::basename(parent_t::m_run_folder); } + /** Get the full path of the baseline output file + * + * @return full path + */ + std::string baseline()const + { + return io::dirname(parent_t::baseline()); + } }; diff --git a/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h b/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h index 741c268c6..d1e87ec09 100644 --- a/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h +++ b/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h @@ -18,17 +18,23 @@ namespace illumina{ namespace interop { namespace unittest /** This generator creates an expected metric set and the corresponding binary data * * @see model::metrics::q_collapsed_metric - * @note Version 2 */ - struct q_collapsed_metric_v2 : metric_test + template + struct q_collapsed_metric_v_2_4 : metric_test { + /** Define a parent type */ + typedef metric_test parent_t; + /** Define a metric set type */ + typedef typename parent_t::metric_set_t metric_set_t; + /** Define a metric type */ + typedef typename parent_t::metric_t metric_t; /** Create the expected metric set * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { - metrics = metric_set_t(VERSION); + metrics = metric_set_t(parent_t::VERSION); metrics.insert(metric_t(1,1105,1,2447414,2334829,2566750,33)); metrics.insert(metric_t(1,1103,1,2436317,2327796,2543605,33)); metrics.insert(metric_t(1,1106,1,2474217,2366046,2583629,33)); @@ -42,7 +48,7 @@ namespace illumina{ namespace interop { namespace unittest { const int tmp[] = { - 2,22 + Version,22 ,1,0,81,4,1,0,54,88,37,0,109,-96,35,0,94,42,39,0,0,0,4,66 ,1,0,79,4,1,0,-35,44,37,0,-12,-124,35,0,-11,-49,38,0,0,0,4,66 ,1,0,82,4,1,0,-23,-64,37,0,94,26,36,0,77,108,39,0,0,0,4,66 @@ -50,14 +56,40 @@ namespace illumina{ namespace interop { namespace unittest buffer.assign(tmp, tmp+util::length_of(tmp)); } }; + /** This generator creates an expected metric set and the corresponding binary data + * + * @see model::metrics::q_collapsed_metric + * @note Version 2 + */ + struct q_collapsed_metric_v2 : q_collapsed_metric_v_2_4<2>{}; + /** This generator creates an expected metric set and the corresponding binary data + * + * @see model::metrics::q_collapsed_metric + * @note Version 3 + */ + struct q_collapsed_metric_v3 : q_collapsed_metric_v_2_4<3>{}; + /** This generator creates an expected metric set and the corresponding binary data + * + * @see model::metrics::q_collapsed_metric + * @note Version 4 + */ + struct q_collapsed_metric_v4 : q_collapsed_metric_v_2_4<4>{}; + /** This generator creates an expected metric set and the corresponding binary data * * @see model::metrics::q_collapsed_metric * @note Version 6 */ - struct q_collapsed_metric_v6 : metric_test + template + struct q_collapsed_metric_v5_6 : metric_test { + /** Define a parent type */ + typedef metric_test parent_t; + /** Define a metric set type */ + typedef typename parent_t::metric_set_t metric_set_t; + /** Define a metric type */ + typedef typename parent_t::metric_t metric_t; enum{ /** Do not check the expected binary data */ disable_binary_data=true, @@ -68,12 +100,13 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { - typedef header_t::qscore_bin_vector_type qscore_bin_vector_type; - typedef header_t::bin_t bin_t; - typedef bin_t::bin_type ushort_t; - typedef metric_t::uint_t uint_t; + typedef typename metric_set_t::header_type header_t; + typedef typename header_t::qscore_bin_vector_type qscore_bin_vector_type; + typedef typename header_t::bin_t bin_t; + typedef typename bin_t::bin_type ushort_t; + typedef typename metric_t::uint_t uint_t; const uint_t bin_count = 7; const ushort_t lower[] = {2, 10, 20, 25, 30, 35, 40}; @@ -83,7 +116,7 @@ namespace illumina{ namespace interop { namespace unittest for(uint_t i=0;i{}; + /** This generator creates an expected metric set and the corresponding binary data + * + * @see model::metrics::q_collapsed_metric + * @note Version 6 + */ + struct q_collapsed_metric_v6 : q_collapsed_metric_v5_6<6>{}; + }}} diff --git a/src/tests/interop/metrics/inc/q_metrics_test.h b/src/tests/interop/metrics/inc/q_metrics_test.h index 51be44511..65f35e405 100644 --- a/src/tests/interop/metrics/inc/q_metrics_test.h +++ b/src/tests/interop/metrics/inc/q_metrics_test.h @@ -27,7 +27,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { metrics = metric_set_t(VERSION); @@ -76,10 +76,11 @@ namespace illumina{ namespace interop { namespace unittest { const size_t lane_count = 1; const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 3, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary[0][0].tile_count(2); summary[0][0].projected_yield_g(0.0098816361278295517); summary[0][0].yield_g(0.0074112270958721638f); @@ -113,7 +114,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { typedef header_t::qscore_bin_vector_type qscore_bin_vector_type; typedef header_t::bin_t bin_t; @@ -175,10 +176,11 @@ namespace illumina{ namespace interop { namespace unittest { const size_t lane_count = 1; const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 2, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary[0][0].tile_count(3); summary[0][0].projected_yield_g(0.0056276721879839897f); summary[0][0].yield_g(0.0056276721879839897f); @@ -212,7 +214,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { typedef header_t::qscore_bin_vector_type qscore_bin_vector_type; typedef header_t::bin_t bin_t; @@ -232,7 +234,6 @@ namespace illumina{ namespace interop { namespace unittest const uint_t hist_all1[] = {0, 267962, 118703, 4284, 2796110, 0, 0}; const uint_t hist_all2[] = {0,241483, 44960, 1100, 2899568, 0 ,0}; const uint_t hist_all3[] = {0,212144, 53942, 427, 2920598, 0, 0}; - std::vector hist_tmp(50, 0); metrics.insert(metric_t(7, 1114, 1, to_vector(hist_all1))); metrics.insert(metric_t(7, 1114, 2, to_vector(hist_all2))); @@ -261,10 +262,11 @@ namespace illumina{ namespace interop { namespace unittest { const size_t lane_count = 1; const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 4, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary[0][0].lane(7); summary[0][0].tile_count(1); summary[0][0].projected_yield_g(0.0095612816512584686f); @@ -298,12 +300,12 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { metrics = metric_set_t(VERSION); typedef metric_t::uint_t uint_t; - std::vector hist_tmp(50, 0); + std::vector hist_tmp(model::metrics::q_metric::MAX_Q_BINS, 0); metrics.insert(metric_t(1, 1110, 1, hist_tmp)); metrics.insert(metric_t(1, 1110, 2, hist_tmp)); diff --git a/src/tests/interop/metrics/inc/tile_metrics_test.h b/src/tests/interop/metrics/inc/tile_metrics_test.h index 450d9d10a..c8eb27370 100644 --- a/src/tests/interop/metrics/inc/tile_metrics_test.h +++ b/src/tests/interop/metrics/inc/tile_metrics_test.h @@ -33,7 +33,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_metric_set(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics) { metrics = metric_set_t(VERSION); @@ -96,11 +96,12 @@ namespace illumina{ namespace interop { namespace unittest { const size_t lane_count = 1; const size_t surface_count = 2; + const size_t channel_count = 2; const model::run::read_info reads[]={ model::run::read_info(1, 1, 3, false), model::run::read_info(2, 1, 3, false) }; - summary.initialize(to_vector(reads), lane_count, surface_count); + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); for(size_t read=0;read index_metric_set; struct index_metrics_tests : public generic_test_fixture< index_metric_set > {}; index_metrics_tests::generator_type index_unit_test_generators[] = { - wrap(new hardcoded_metric_generator< index_metric_v1 >) , + wrap(new hardcoded_metric_generator< index_metric_v1 >), wrap(new write_read_metric_generator< index_metric_v1 >) }; @@ -42,7 +42,8 @@ INSTANTIATE_TEST_CASE_P(index_metric_unit_test, TEST_P(index_metrics_tests, test_read_write) { typedef index_metric_set::const_iterator const_iterator; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); EXPECT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); @@ -59,7 +60,7 @@ TEST_P(index_metrics_tests, test_read_write) EXPECT_EQ(it_expected->indices(i).index_seq(), it_actual->indices(i).index_seq()); EXPECT_EQ(it_expected->indices(i).sample_id(), it_actual->indices(i).sample_id()); EXPECT_EQ(it_expected->indices(i).sample_proj(), it_actual->indices(i).sample_proj()); - EXPECT_EQ(it_expected->indices(i).count(), it_actual->indices(i).count()); + EXPECT_EQ(it_expected->indices(i).cluster_count(), it_actual->indices(i).cluster_count()); } } } @@ -68,7 +69,7 @@ TEST_P(index_metrics_tests, test_read_write) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator index_regression_gen("metrics"); +regression_test_metric_generator index_regression_gen; INSTANTIATE_TEST_CASE_P(index_metric_regression_test, index_metrics_tests, ProxyValuesIn(index_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/metric_stream_error_test.cpp b/src/tests/interop/metrics/metric_stream_error_test.cpp new file mode 100644 index 000000000..07bbd2111 --- /dev/null +++ b/src/tests/interop/metrics/metric_stream_error_test.cpp @@ -0,0 +1,137 @@ +/** Unit tests for the metric stream error coniditions + * + * + * @file + * @date 11/20/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#ifdef _MSC_VER +#pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants +#endif + +#include +#include "interop/io/metric_stream.h" +#include "interop/io/metric_file_stream.h" +#include "src/tests/interop/metrics/inc/metric_format_fixtures.h" + +using namespace illumina::interop; +using namespace illumina::interop::unittest; + + +/** Fixture for expected vs actual binary data */ +template +struct metric_stream_error_test : public ::testing::Test, public TestSetup +{ + /** Type of metric set */ + typedef typename TestSetup::metric_set_t metric_set_t; + /** Constructor */ + metric_stream_error_test() + { + TestSetup::create_binary_data(expected); + } + /** Expected binary data */ + std::string expected; +}; + +TYPED_TEST_CASE_P(metric_stream_error_test); + +/** Confirm bad_format_exception is thrown when version is unsupported + */ +TYPED_TEST_P(metric_stream_error_test, test_hardcoded_bad_format_exception) +{ + std::string tmp = std::string(TestFixture::expected); + tmp[0] = 34; + typename TypeParam::metric_set_t metrics; + EXPECT_THROW(io::read_interop_from_string(tmp, metrics), io::bad_format_exception); +} + + +/** Confirm incomplete_file_exception is thrown for a small partial record + */ +TYPED_TEST_P(metric_stream_error_test, test_hardcoded_incomplete_file_exception) +{ + ::uint32_t incomplete = 0; + for (::uint32_t i = 2; i < 25; i++) + { + typename TypeParam::metric_set_t metrics; + try + { + io::read_interop_from_string(TestFixture::expected.substr(0, i), metrics); + } + catch (const io::incomplete_file_exception &) + { incomplete++; } + catch (const std::exception &ex) + { + std::cerr << i << " " << ex.what() << std::endl; + throw; + } + } + EXPECT_TRUE(incomplete > 10) << "incomplete: " << incomplete; +} +/** Confirm incomplete_file_exception is thrown for a mostly complete file + */ +TYPED_TEST_P(metric_stream_error_test, test_hardcoded_incomplete_file_exception_last_metric) +{ + typename TypeParam::metric_set_t metrics; + EXPECT_THROW(io::read_interop_from_string( + TestFixture::expected.substr(0, TestFixture::expected.length() - 4), metrics), + io::incomplete_file_exception); +} + +// TODO: Add write header test + +/** Confirm bad_format_exception is thrown when record size is incorrect + */ +TYPED_TEST_P(metric_stream_error_test, test_hardcoded_incorrect_record_size) +{ + if (TypeParam::disable_check_record_size) return; + std::string tmp = std::string(TestFixture::expected); + tmp[1] = 0; + tmp[2] = 0; + typename TypeParam::metric_set_t metrics; + EXPECT_THROW(io::read_interop_from_string(tmp, metrics), io::bad_format_exception); +} +/** Confirm file_not_found_exception is thrown when a file is not found + */ +TYPED_TEST_P(metric_stream_error_test, test_hardcoded_file_not_found) +{ + typename TypeParam::metric_set_t metrics; + EXPECT_THROW(io::read_interop("/NO/FILE/EXISTS", metrics), io::file_not_found_exception); +} +/** Confirm reading from good data does not throw an exception + */ +TYPED_TEST_P(metric_stream_error_test, test_hardcoded_read) +{ + std::string tmp = std::string(TestFixture::expected); + typename TypeParam::metric_set_t metrics; + EXPECT_NO_THROW(io::read_interop_from_string(tmp, metrics)); +} + + +/** Confirm that the InterOp writer will not access memory out of bounds + */ +TEST(metric_stream_error_test, image_metric_out_of_bounds) +{ + typedef model::metrics::image_metric::ushort_t ushort_t; + model::metric_base::metric_set metric_set( + model::metrics::image_metric_header(2)); + const ushort_t min_vals[] = {100, 200, 300}; + metric_set.insert( model::metrics::image_metric(1, 1101, 1, 2, util::to_vector(min_vals), util::to_vector(min_vals)) ); + std::ostringstream fout; + EXPECT_THROW(io::write_metrics(fout, metric_set), io::bad_format_exception); +} + +REGISTER_TYPED_TEST_CASE_P(metric_stream_error_test, + test_hardcoded_bad_format_exception, + test_hardcoded_incomplete_file_exception, + test_hardcoded_incomplete_file_exception_last_metric, + test_hardcoded_incorrect_record_size, + test_hardcoded_file_not_found, + test_hardcoded_read +); + + +INSTANTIATE_TYPED_TEST_CASE_P(Public, metric_stream_error_test, PublicFormats); + diff --git a/src/tests/interop/metrics/metric_streams_test.cpp b/src/tests/interop/metrics/metric_streams_test.cpp index b6f4865d1..20660161e 100644 --- a/src/tests/interop/metrics/metric_streams_test.cpp +++ b/src/tests/interop/metrics/metric_streams_test.cpp @@ -12,6 +12,8 @@ #endif #include +#include "interop/io/metric_stream.h" +#include "interop/io/metric_file_stream.h" #include "src/tests/interop/metrics/inc/metric_format_fixtures.h" using namespace illumina::interop; @@ -29,7 +31,7 @@ struct metric_stream_test : public ::testing::Test, public TestSetup { TestSetup::create_binary_data(expected); metric_set_t metrics; - TestSetup::create_metric_set(metrics); + TestSetup::create_expected(metrics); std::ostringstream fout; io::write_metrics(fout, metrics); actual = fout.str(); @@ -55,82 +57,44 @@ TYPED_TEST_P(metric_stream_test, test_write_read_binary_data) } } -/** Confirm bad_format_exception is thrown when version is unsupported +/** Confirm the header size matches what is read */ -TYPED_TEST_P(metric_stream_test, test_hardcoded_bad_format_exception) +TYPED_TEST_P(metric_stream_test, test_header_size) { std::string tmp = std::string(TestFixture::expected); - tmp[0] = 34; typename TypeParam::metric_set_t metrics; - EXPECT_THROW(io::read_interop_from_string(tmp, metrics), io::bad_format_exception); + const size_t actual_size = io::read_header_from_string(tmp, metrics); + const size_t expected_size = io::header_size(metrics); + EXPECT_EQ(expected_size, actual_size); } - -/** Confirm incomplete_file_exception is thrown for a small partial record - */ -TYPED_TEST_P(metric_stream_test, test_hardcoded_incomplete_file_exception) -{ - ::uint32_t incomplete = 0; - for (::uint32_t i = 2; i < 25; i++) - { - typename TypeParam::metric_set_t metrics; - try - { - io::read_interop_from_string(TestFixture::expected.substr(0, i), metrics); - } - catch (const io::incomplete_file_exception &) - { incomplete++; } - catch (const std::exception &ex) - { - std::cerr << i << " " << ex.what() << std::endl; - throw; - } - } - EXPECT_TRUE(incomplete > 10) << "incomplete: " << incomplete; -} -/** Confirm incomplete_file_exception is thrown for a mostly complete file - */ -TYPED_TEST_P(metric_stream_test, test_hardcoded_incomplete_file_exception_last_metric) -{ - typename TypeParam::metric_set_t metrics; - EXPECT_THROW(io::read_interop_from_string( - TestFixture::expected.substr(0, TestFixture::expected.length() - 4), metrics), - io::incomplete_file_exception); -} -/** Confirm bad_format_exception is thrown when record size is incorrect +/** Confirm the header size matches what is read */ -TYPED_TEST_P(metric_stream_test, test_hardcoded_incorrect_record_size) +TYPED_TEST_P(metric_stream_test, test_read_data_size) { - if (TypeParam::disable_check_record_size) return; + if (TypeParam::disable_binary_data_size || TypeParam::disable_binary_data) return; std::string tmp = std::string(TestFixture::expected); - tmp[1] = 0; - tmp[2] = 0; typename TypeParam::metric_set_t metrics; - EXPECT_THROW(io::read_interop_from_string(tmp, metrics), io::bad_format_exception); + io::read_interop_from_string(tmp, metrics); + if(io::is_multi_record(metrics)) return; + const size_t expected_size = io::compute_buffer_size(metrics); + EXPECT_EQ(tmp.size(), expected_size); } -/** Confirm file_not_found_exception is thrown when a file is not found - */ -TYPED_TEST_P(metric_stream_test, test_hardcoded_file_not_found) -{ - typename TypeParam::metric_set_t metrics; - EXPECT_THROW(io::read_interop("/NO/FILE/EXISTS", metrics), io::file_not_found_exception); -} -/** Confirm reading from good data does not throw an exception - */ -TYPED_TEST_P(metric_stream_test, test_hardcoded_read) + +TEST(metric_stream_test, list_filenames) { - std::string tmp = std::string(TestFixture::expected); - typename TypeParam::metric_set_t metrics; - EXPECT_NO_THROW(io::read_interop_from_string(tmp, metrics)); + std::vector error_metric_files; + io::list_interop_filenames< model::metrics::error_metric >(error_metric_files, ""); + ASSERT_EQ(error_metric_files.size(), 1u); + EXPECT_EQ(error_metric_files[0], io::combine("InterOp", "ErrorMetricsOut.bin")); } -REGISTER_TYPED_TEST_CASE_P(metric_stream_test, test_write_read_binary_data, - test_hardcoded_bad_format_exception, - test_hardcoded_incomplete_file_exception, - test_hardcoded_incomplete_file_exception_last_metric, - test_hardcoded_incorrect_record_size, - test_hardcoded_file_not_found, - test_hardcoded_read + + +REGISTER_TYPED_TEST_CASE_P(metric_stream_test, + test_read_data_size, + test_header_size, + test_write_read_binary_data ); diff --git a/src/tests/interop/metrics/q_by_lane_metric_test.cpp b/src/tests/interop/metrics/q_by_lane_metric_test.cpp index c08eca21c..8fe4ef1b1 100644 --- a/src/tests/interop/metrics/q_by_lane_metric_test.cpp +++ b/src/tests/interop/metrics/q_by_lane_metric_test.cpp @@ -77,7 +77,7 @@ TEST(run_metrics_q_by_lane_test, test_is_group_empty) std::string data; q_metric_v4::create_binary_data(data); io::read_interop_from_string(data, - metrics.get_set()); - logic::metric::create_q_metrics_by_lane(metrics.get_set(), metrics.get_set()); + metrics.get()); + logic::metric::create_q_metrics_by_lane(metrics.get(), metrics.get()); EXPECT_FALSE(metrics.is_group_empty(constants::QByLane)); } diff --git a/src/tests/interop/metrics/q_collapsed_metrics_test.cpp b/src/tests/interop/metrics/q_collapsed_metrics_test.cpp index a3780b84a..ac74c9f1d 100644 --- a/src/tests/interop/metrics/q_collapsed_metrics_test.cpp +++ b/src/tests/interop/metrics/q_collapsed_metrics_test.cpp @@ -29,9 +29,15 @@ struct q_collapsed_metrics_tests : public generic_test_fixture< q_collapsed_metr q_collapsed_metrics_tests::generator_type q_collapsed_unit_test_generators[] = { - wrap(new hardcoded_metric_generator< q_collapsed_metric_v2 >) , + wrap(new hardcoded_metric_generator< q_collapsed_metric_v2 >), wrap(new write_read_metric_generator< q_collapsed_metric_v2 >), - wrap(new hardcoded_metric_generator< q_collapsed_metric_v6 >) , + wrap(new hardcoded_metric_generator< q_collapsed_metric_v3 >), + wrap(new write_read_metric_generator< q_collapsed_metric_v3 >), + wrap(new hardcoded_metric_generator< q_collapsed_metric_v4 >), + wrap(new write_read_metric_generator< q_collapsed_metric_v4 >), + wrap(new hardcoded_metric_generator< q_collapsed_metric_v5 >), + wrap(new write_read_metric_generator< q_collapsed_metric_v5 >), + wrap(new hardcoded_metric_generator< q_collapsed_metric_v6 >), wrap(new write_read_metric_generator< q_collapsed_metric_v6 >) }; @@ -50,7 +56,8 @@ INSTANTIATE_TEST_CASE_P(q_collapsed_metric_unit_test, TEST_P(q_collapsed_metrics_tests, test_read_write) { typedef q_collapsed_metric_set::const_iterator const_iterator; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); EXPECT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); EXPECT_EQ(actual.max_cycle(), expected.max_cycle()); @@ -109,7 +116,7 @@ TEST(q_collapsed_metrics_test, test_convert_write_read) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator q_collapsed_regression_gen("metrics"); +regression_test_metric_generator q_collapsed_regression_gen; INSTANTIATE_TEST_CASE_P(q_collapsed_metric_regression_test, q_collapsed_metrics_tests, ProxyValuesIn(q_collapsed_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/q_metrics_test.cpp b/src/tests/interop/metrics/q_metrics_test.cpp index 7abf4cecc..93f3843ca 100644 --- a/src/tests/interop/metrics/q_metrics_test.cpp +++ b/src/tests/interop/metrics/q_metrics_test.cpp @@ -54,7 +54,8 @@ INSTANTIATE_TEST_CASE_P(q_metric_unit_test, TEST_P(q_metrics_tests, test_read_write) { typedef q_metric_set::const_iterator const_iterator; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); EXPECT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); EXPECT_EQ(actual.max_cycle(), expected.max_cycle()); @@ -89,10 +90,10 @@ TEST_P(q_metrics_tests, test_read_write) TEST(q_metrics_test, test_populate_cumulative_on_empty) { q_metric_set actual; - q_metric_v6::create_metric_set(actual); + q_metric_v6::create_expected(actual); logic::metric::populate_cumulative_distribution(actual); - for(q_metric_set::const_iterator cur=actual.metrics().begin();cur != actual.metrics().end();++cur) + for(q_metric_set::const_iterator cur=actual.begin();cur != actual.end();++cur) EXPECT_TRUE(!cur->is_cumulative_empty()); metric_set empty_metrics; logic::metric::populate_cumulative_distribution(empty_metrics); @@ -131,7 +132,7 @@ TEST(q_metrics_test, test_cumulative) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator q_regression_gen("metrics"); +regression_test_metric_generator q_regression_gen; INSTANTIATE_TEST_CASE_P(q_metric_regression_test, q_metrics_tests, ProxyValuesIn(q_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/run_metric_test.cpp b/src/tests/interop/metrics/run_metric_test.cpp index 356e9eadb..7e321cc5e 100644 --- a/src/tests/interop/metrics/run_metric_test.cpp +++ b/src/tests/interop/metrics/run_metric_test.cpp @@ -24,7 +24,7 @@ struct run_metric_test : public ::testing::Test, public TestSetup /** Constructor */ run_metric_test() { - TestSetup::create_metric_set(expected.template get()); + TestSetup::create_expected(expected.template get()); } /** Expected metric set */ model::metrics::run_metrics expected; @@ -39,7 +39,7 @@ TYPED_TEST_P(run_metric_test, test_clear) metric_set_t& metric_set = TestFixture::expected.template get(); size_t cnt = metric_set.size(); TestFixture::expected.clear(); - TypeParam::create_metric_set(metric_set); + TypeParam::create_expected(metric_set); EXPECT_EQ(cnt, metric_set.size()); } @@ -74,6 +74,7 @@ TYPED_TEST_P(run_metric_test, test_expected_get_metric) } } + REGISTER_TYPED_TEST_CASE_P(run_metric_test, test_clear, is_group_empty_true, diff --git a/src/tests/interop/metrics/tile_metrics_test.cpp b/src/tests/interop/metrics/tile_metrics_test.cpp index 1f92a3ae7..8fde30f5b 100644 --- a/src/tests/interop/metrics/tile_metrics_test.cpp +++ b/src/tests/interop/metrics/tile_metrics_test.cpp @@ -26,6 +26,53 @@ using namespace illumina::interop::unittest; using namespace illumina; +/** Generate the actual metric set by reading in from hardcoded binary buffer + * + * The expected metric set is provided by the generator. + */ +template +class write_metric_generator : public abstract_generator< typename Gen::metric_set_t > +{ + typedef typename Gen::metric_set_t metric_set_t; + typedef typename abstract_generator::parent_type parent_t; +public: + /** Generate the expected and actual metric sets + * + * @param expected expected metric set + * @param actual actual metric set + */ + ::testing::AssertionResult generate(metric_set_t& expected, metric_set_t& actual, bool*)const + { + expected.clear(); + tile_metric expected_metric(1,1101); + expected.set_version(Gen::VERSION); + expected.insert(expected_metric); + + std::ostringstream fout; + illumina::interop::io::write_metrics(fout, expected); + actual.clear(); + + try{ + io::read_interop_from_string(fout.str(), actual); + }catch(const io::incomplete_file_exception&){} + return ::testing::AssertionSuccess(); + } + /** Create a copy of this object + * + * @return pointer to copy + */ + parent_t clone()const{return new write_metric_generator;} + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream& out)const + { + out << "write_metric_generator<" << Gen::name() << ">"; + } + +}; + typedef metric_set< tile_metric > tile_metric_set; /** Setup for tests that compare two tile metric sets */ struct tile_metrics_tests : public generic_test_fixture< tile_metric_set > {}; @@ -33,7 +80,8 @@ struct tile_metrics_tests : public generic_test_fixture< tile_metric_set > {}; tile_metrics_tests::generator_type tile_unit_test_generators[] = { wrap(new hardcoded_metric_generator< tile_metric_v2 >) , - wrap(new write_read_metric_generator< tile_metric_v2 >) + wrap(new write_read_metric_generator< tile_metric_v2 >), + wrap(new write_metric_generator< tile_metric_v2 >) }; // Setup unit tests for tile_metrics_tests @@ -50,7 +98,8 @@ TEST_P(tile_metrics_tests, test_read_write) { typedef tile_metric_set::const_iterator const_iterator; typedef tile_metric_set::metric_type metric_t; - if(!test) return;// Disable test for rebaseline + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); const float scale = (test_modifier==2) ? 0.01f : ( (test_modifier==1) ? 100.0f : 1.0f); ASSERT_EQ(actual.version(), expected.version()); ASSERT_EQ(actual.size(), expected.size()); @@ -63,10 +112,10 @@ TEST_P(tile_metrics_tests, test_read_write) EXPECT_EQ(it_expected->lane(), it_actual->lane()); EXPECT_EQ(it_expected->tile(), it_actual->tile()); - EXPECT_NEAR(it_expected->cluster_density(), it_actual->cluster_density(), tol); - EXPECT_NEAR(it_expected->cluster_density_pf(), it_actual->cluster_density_pf(), tol); - EXPECT_NEAR(it_expected->cluster_count(), it_actual->cluster_count(), tol); - EXPECT_NEAR(it_expected->cluster_count_pf(), it_actual->cluster_count_pf(), tol); + INTEROP_EXPECT_NEAR(it_expected->cluster_density(), it_actual->cluster_density(), tol); + INTEROP_EXPECT_NEAR(it_expected->cluster_density_pf(), it_actual->cluster_density_pf(), tol); + INTEROP_EXPECT_NEAR(it_expected->cluster_count(), it_actual->cluster_count(), tol); + INTEROP_EXPECT_NEAR(it_expected->cluster_count_pf(), it_actual->cluster_count_pf(), tol); EXPECT_EQ(it_expected->read_metrics().size(), it_actual->read_metrics().size()); for(metric_t::read_metric_vector::const_iterator it_read_expected = it_expected->read_metrics().begin(), it_read_actual = it_actual->read_metrics().begin(); @@ -74,16 +123,14 @@ TEST_P(tile_metrics_tests, test_read_write) it_read_actual != it_actual->read_metrics().end(); it_read_expected++, it_read_actual++) { EXPECT_EQ(it_read_expected->read(), it_read_actual->read()); - if(!std::isnan(it_read_expected->percent_aligned()) || !std::isnan(it_read_actual->percent_aligned())) - EXPECT_NEAR(it_read_expected->percent_aligned(), it_read_actual->percent_aligned(), tol); - if(!std::isnan(it_read_expected->percent_phasing()) || !std::isnan(it_read_actual->percent_phasing())) - EXPECT_NEAR(it_read_expected->percent_phasing()*scale, it_read_actual->percent_phasing(), tol); - if(!std::isnan(it_read_expected->percent_prephasing()) || !std::isnan(it_read_actual->percent_prephasing())) - EXPECT_NEAR(it_read_expected->percent_prephasing()*scale, it_read_actual->percent_prephasing(), tol); + INTEROP_ASSERT_NEAR(it_read_expected->percent_aligned(), it_read_actual->percent_aligned(), tol); + INTEROP_ASSERT_NEAR(it_read_expected->percent_phasing()*scale, it_read_actual->percent_phasing(), tol); + INTEROP_ASSERT_NEAR(it_read_expected->percent_prephasing()*scale, it_read_actual->percent_prephasing(), tol); } } } + TEST(tile_metrics_test, test_unique_id_four_digit) { typedef metric_set::uint_t uint_t; @@ -159,7 +206,7 @@ TEST(tile_metrics_test, test_tile_metric_for_lane) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -regression_test_metric_generator tile_regression_gen("metrics"); +regression_test_metric_generator tile_regression_gen; INSTANTIATE_TEST_CASE_P(tile_metric_regression_test, tile_metrics_tests, ProxyValuesIn(tile_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/unit_tests.cpp b/src/tests/interop/unit_tests.cpp index 39ded10a1..d13d9d346 100644 --- a/src/tests/interop/unit_tests.cpp +++ b/src/tests/interop/unit_tests.cpp @@ -53,12 +53,15 @@ int main(int argc, char **argv) return 1; } - // remove the default listener - ::testing::TestEventListeners& listeners = ::testing::UnitTest::GetInstance()->listeners(); - ::testing::TestEventListener *default_printer = listeners.Release(listeners.default_result_printer()); - // Add listener that will only print failures - failure_listener *listener = new failure_listener(default_printer); - listeners.Append(listener); + + { + // remove the default listener + ::testing::TestEventListeners &listeners = ::testing::UnitTest::GetInstance()->listeners(); + ::testing::TestEventListener *default_printer = listeners.Release(listeners.default_result_printer()); + // Add listener that will only print failures + failure_listener *listener = new failure_listener(default_printer); + listeners.Append(listener); + } // run try diff --git a/src/tests/interop/util/stat_test.cpp b/src/tests/interop/util/stat_test.cpp index e8210e967..45ef432b4 100644 --- a/src/tests/interop/util/stat_test.cpp +++ b/src/tests/interop/util/stat_test.cpp @@ -26,7 +26,7 @@ typedef metric_set< tile_metric > tile_metric_set; TEST(stat_test, median) { tile_metric_set expected; - tile_metric_v2::create_metric_set(expected); + tile_metric_v2::create_expected(expected); const float tol = 1e-7f / 0.01f; const size_t read = 0; float expected_percent_aligned_avg = interop::util::median(expected.begin(), @@ -40,7 +40,7 @@ TEST(stat_test, median) TEST(stat_test, mean) { tile_metric_set expected; - tile_metric_v2::create_metric_set(expected); + tile_metric_v2::create_expected(expected); const float tol = 1e-7f / 0.01f; const size_t read = 0; float expected_percent_aligned_avg = interop::util::mean(expected.begin(), @@ -53,7 +53,7 @@ TEST(stat_test, mean) TEST(stat_test, nan_mean) { tile_metric_set expected; - tile_metric_v2::create_metric_set(expected); + tile_metric_v2::create_expected(expected); const float tol = 1e-7f / 0.01f; const size_t read = 0; float expected_percent_aligned_avg = interop::util::nan_mean(expected.begin(), @@ -67,7 +67,7 @@ TEST(stat_test, nan_mean) TEST(stat_test, standard_deviation) { tile_metric_set expected; - tile_metric_v2::create_metric_set(expected); + tile_metric_v2::create_expected(expected); const float tol = 1e-7f / 0.01f; const size_t read = 0; float expected_percent_aligned_std = std::sqrt(interop::util::variance(expected.begin(), @@ -81,7 +81,7 @@ TEST(stat_test, standard_deviation) TEST(stat_test, nan_standard_deviation) { tile_metric_set expected; - tile_metric_v2::create_metric_set(expected); + tile_metric_v2::create_expected(expected); const float tol = 1e-7f / 0.01f; const size_t read = 0; float expected_percent_aligned_std = std::sqrt(interop::util::nan_variance(expected.begin(), @@ -95,7 +95,7 @@ TEST(stat_test, nan_standard_deviation) TEST(stat_test, standard_deviation_vec) { tile_metric_set expected; - tile_metric_v2::create_metric_set(expected); + tile_metric_v2::create_expected(expected); const float tol = 1e-7f / 0.01f; const size_t read = 0; std::vector percent_aligned_vec(expected.size()); @@ -105,3 +105,4 @@ TEST(stat_test, standard_deviation_vec) EXPECT_NEAR(expected_percent_aligned_std, 0.074578315019607544, tol); } + diff --git a/tools/build_test.bat b/tools/build_test.bat index 030a12cdb..ca69fa553 100644 --- a/tools/build_test.bat +++ b/tools/build_test.bat @@ -82,11 +82,6 @@ cmake --build . --target check --config %BUILD_TYPE% -- /M if %errorlevel% neq 0 exit /b %errorlevel% echo ##teamcity[blockClosed name='Test %BUILD_TYPE% Visual Studio 2015 Win64'] -echo ##teamcity[blockOpened name='NuSpec Debug Visual Studio 2015 Win64'] -cmake --build . --target nuspec --config Debug -- /M -if %errorlevel% neq 0 exit /b %errorlevel% -echo ##teamcity[blockClosed name='NuSpec Debug Visual Studio 2015 Win64'] - echo ##teamcity[blockOpened name='NuSpec Release Visual Studio 2015 Win64'] cmake --build . --target nuspec --config Release -- /M if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/tools/hooks/pre-commit.sh b/tools/hooks/pre-commit.sh index 71ab17ba4..e4ff02c24 100755 --- a/tools/hooks/pre-commit.sh +++ b/tools/hooks/pre-commit.sh @@ -9,8 +9,15 @@ if [[ "$BRANCH" == "master" ]]; then exit 1 fi +# This is old code, which did not work for Git versions later than 1.7 for updating a version file using +# the pre-commit hook +# +# The code was moved to the pre-push hook +# +# This tests whether there are any stashed changes waiting to be added #git diff --cached --exit-code --quiet #status1=$? +# This tests whether there are any changes waiting to be added #git diff --exit-code --quiet #status2=$? #if [ "$status1" == "1" ] || ["$status2" == "1"] ; then diff --git a/tools/package.bat b/tools/package.bat new file mode 100644 index 000000000..af92a4f22 --- /dev/null +++ b/tools/package.bat @@ -0,0 +1,57 @@ +echo off +setlocal enabledelayedexpansion +rem -------------------------------------------------------------------------------------------------------------------- +rem Package InterOp for Windows Systems +rem +rem The script takes two arguments: +rem 1. Path to third party binaries: E.g. GTest, NUnit +rem +rem Example Running script (from source directory) +rem .\tools\build_test.bat c:\external +rem +rem Note, you must already have CMake, MinGW and Visual Studio installed and on your path. +rem +rem -------------------------------------------------------------------------------------------------------------------- + +rem -------------------------------------------------------------------------------------------------------------------- +rem MinGW Build Test Script +rem -------------------------------------------------------------------------------------------------------------------- + +set SOURCE_DIR=..\ +set BUILD_PARAM= +set BUILD_TYPE=Release +set BUILD_PATH=%1% +if NOT "%1" == "" ( +set BUILD_PARAM=-DGTEST_ROOT=%BUILD_PATH% -DJUNIT_ROOT=%BUILD_PATH% -DGMOCK_ROOT=%BUILD_PATH% -DNUNIT_ROOT=%BUILD_PATH%/NUnit-2.6.4 +) + + + +echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] +mkdir build_vs2015_x64_%BUILD_TYPE% +cd build_vs2015_x64_%BUILD_TYPE% +echo cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] + +echo ##teamcity[blockOpened name='Build %BUILD_TYPE% Visual Studio 2015 Win64'] +cmake --build . --config %BUILD_TYPE% -- /M +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Build %BUILD_TYPE% Visual Studio 2015 Win64'] + +echo ##teamcity[blockOpened name='Test %BUILD_TYPE% Visual Studio 2015 Win64'] +cmake --build . --target check --config %BUILD_TYPE% -- /M +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Test %BUILD_TYPE% Visual Studio 2015 Win64'] + +echo ##teamcity[blockOpened name='Package Release Visual Studio 2015 Win64'] +cmake --build . --target package --config Release -- /M +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Package Release Visual Studio 2015 Win64'] + +echo ##teamcity[blockOpened name='NuSpec Release Visual Studio 2015 Win64'] +cmake --build . --target nuspec --config Release -- /M +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='NuSpec Release Visual Studio 2015 Win64'] + diff --git a/tools/package.sh b/tools/package.sh new file mode 100644 index 000000000..50286a0ba --- /dev/null +++ b/tools/package.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +######################################################################################################################## +# Package InterOp for Deployment +# +# This script takes three parameters: +# 1. Path to third party binaries: E.g. GTest, NUnit +# 2. Package suffix +# 3. Enable C++98 +# +# Example running script (from the source directory) +# +# sh tools/build_test.sh /var/external_libs OFF centos7 +# +# Note, you must already have CMake, GCC and nuget installed and on your path. +# +######################################################################################################################## + +# Ensure the script stops on first error +set -e + + +source_dir="../" +build_param="" + +build_type="Linux" + +if [ ! -z $1 ] ; then + build_param="-DJUNIT_ROOT=$1 -DGTEST_ROOT=$1 -DGMOCK_ROOT=$1 -DNUNIT_ROOT=$1/NUnit-2.6.4" +fi + +if [ ! -z $2 ] ; then + build_param="$build_param -DENABLE_BACKWARDS_COMPATIBILITY=$2" +fi + +if [ ! -z $3 ] ; then + build_param="$build_param -DPACKAGE_SUFFIX=$3" +fi + +if [ -e /opt/rh/devtoolset-2/root/usr/bin/g++ ] ; then + export CXX=/opt/rh/devtoolset-2/root/usr/bin/g++ + export CC=/opt/rh/devtoolset-2/root/usr/bin/gcc +fi + +echo "##teamcity[blockOpened name='Configure $build_type']" +mkdir build_${build_type} +cd build_${build_type} +echo "cmake $source_dir -DCMAKE_BUILD_TYPE=Release $build_param" +cmake $source_dir -DCMAKE_BUILD_TYPE=Release $build_param +echo "##teamcity[blockClosed name='Configure $build_type']" + +echo "##teamcity[blockOpened name='Test $build_type']" +cmake --build . --target check -- -j 8 +echo "##teamcity[blockClosed name='Test $build_type']" + +echo "##teamcity[blockOpened name='Build $build_type']" +cmake --build . -- -j 8 +echo "##teamcity[blockClosed name='Build $build_type']" + +echo "##teamcity[blockOpened name='Package $build_type']" +cmake --build . --target package -- -j 8 +echo "##teamcity[blockClosed name='Package $build_type']" + +echo "##teamcity[blockOpened name='NuSpec Creation $build_type']" +cmake --build . --target nuspec -- -j 8 +echo "##teamcity[blockClosed name='NuSpec Creation $build_type']" + +echo "##teamcity[blockOpened name='NuPack $build_type']" +nuget pack src/ext/csharp/package.nuspec +echo "##teamcity[blockClosed name='NuPack $build_type']" + +cd .. diff --git a/tools/regression_test.ps1 b/tools/regression_test.ps1 new file mode 100644 index 000000000..712bd89bf --- /dev/null +++ b/tools/regression_test.ps1 @@ -0,0 +1,91 @@ +# -------------------------------------------------------------------------------------------------------------------- +# Regression test for MS Visual Studio Compiler +# +# The script takes 4 arguments: +# 1. Build Configuration: Debug or Release +# 2. Path to third party binaries: E.g. GTest, NUnit +# 3. Path to regression test data +# 4. Path to baseline data +# +# Example Running script (from source directory) +# .\tools\build_test.bat -config Debug -lib_path c:\external -data_path c:\RegressionData -baseline_path c:\BaselineData +# +# Note, you must already have CMake and Visual Studio installed and on your path. +# +# -------------------------------------------------------------------------------------------------------------------- + + +param( + [string]$config="Debug", + [string]$lib_path="", + [Parameter()][ValidateNotNullOrEmpty()][string]$data_path=$(throw "Regression data path must be specified"), + [Parameter()][ValidateNotNullOrEmpty()][string]$baseline_path=$(throw "Regression baseline path must be specified"), + [string]$source_path="../", + [string]$generator="`"Visual Studio 14 2015 Win64`"", + [switch]$rebaseline = $false +) + +$build_param="" +if($lib_path) { $build_param="-DGTEST_ROOT=$lib_path -DJUNIT_ROOT=$lib_path -DGMOCK_ROOT=$lib_path -DNUNIT_ROOT=$lib_path/NUnit-2.6.4"} + +# -------------------------------------------------------------------------------------------------------------------- +Write-Host "##teamcity[blockOpened name='Configure $config $generator']" +new-item build_vs2015_x64_$config -itemtype directory +set-location -path build_vs2015_x64_$config +Write-Host "cmake $source_path -G $generator -DCMAKE_BUILD_TYPE=$config $build_param -DFORCE_SHARED_CRT=OFF" +cmake $source_path -G $generator -DCMAKE_BUILD_TYPE=$config $build_param -DFORCE_SHARED_CRT=OFF +$test_code=$lastexitcode +Write-Host "##teamcity[blockClosed name='Configure $config $generator']" +if ($test_code -ne 0) +{ + cd .. + exit $test_code +} + +# -------------------------------------------------------------------------------------------------------------------- + +Write-Host "##teamcity[blockOpened name='Build $config $generator']" +& cmake --build . --config $config --target interop_gtests +$test_code=$lastexitcode +Write-Host "##teamcity[blockClosed name='Build $config $generator']" +if ($test_code -ne 0) +{ + cd .. + exit $test_code +} + +$datasets = Get-ChildItem $data_path | foreach {$_.fullname} +if($rebaseline) +{ + $backup_baseline=$baseline_path + "_last" + if(Test-Path -Path $backup_baseline) + { + Remove-Item -Recurse -Force $backup_baseline + } + Move-Item $baseline_path $backup_baseline + Write-Host "##teamcity[blockOpened name='Rebaseline $config $generator']" + & src\tests\interop\$config\interop_gtests.exe $baseline_path $datasets --rebaseline + $test_code=$lastexitcode + Write-Host "##teamcity[blockClosed name='Test $config $generator']" + if ($test_code -ne 0) + { + cd .. + Write-Host "##teamcity[buildStatus status='FAILURE' text='Rebaseline failed!']" + exit 1 + } +} + +Write-Host "##teamcity[blockOpened name='Test $config $generator']" +& src\tests\interop\$config\interop_gtests.exe $baseline_path $datasets +$test_code=$lastexitcode +Write-Host "##teamcity[blockClosed name='Test $config $generator']" +if ($test_code -ne 0) +{ + cd .. + Write-Host "##teamcity[buildStatus status='FAILURE' text='Not all regression tests passed!']" + exit 1 +} + +cd .. + + From 73c1df46be44d1d8132dc25ffcad2241046b04a6 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Wed, 7 Dec 2016 11:48:48 -0800 Subject: [PATCH 2/4] IPA-5869: Add section filtering --- cmake/package.nuspec.in | 2 +- interop/model/plot/filter_options.h | 21 ++++++++++++-- interop/model/run/flowcell_layout.h | 29 +++++++++++++++++-- src/ext/csharp/CMakeLists.txt | 7 ++++- src/interop/model/run/info.cpp | 2 +- .../logic/inc/metric_filter_iterator.h | 4 ++- 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/cmake/package.nuspec.in b/cmake/package.nuspec.in index 7a179ce56..7408fcfa7 100644 --- a/cmake/package.nuspec.in +++ b/cmake/package.nuspec.in @@ -26,7 +26,7 @@ - + diff --git a/interop/model/plot/filter_options.h b/interop/model/plot/filter_options.h index 5620874d6..bbfe35978 100644 --- a/interop/model/plot/filter_options.h +++ b/interop/model/plot/filter_options.h @@ -128,10 +128,10 @@ namespace illumina { namespace interop { namespace model { namespace plot INTEROP_THROW(model::invalid_filter_option, "Swath number exceeds total number of swaths: " << m_swath << " > " << run_info.flowcell().swath_count()); - if(!all_sections() && m_section > run_info.flowcell().sections_per_lane()) + if(!all_sections() && m_section > run_info.flowcell().total_number_of_sections()) INTEROP_THROW(model::invalid_filter_option, "Section number exceeds total number of sections: " - << m_section << " > " << run_info.flowcell().sections_per_lane()); + << m_section << " > " << run_info.flowcell().total_number_of_sections()); if(logic::utils::is_base_metric(type)) { if(!all_bases() && (m_base >= static_cast(constants::NUM_OF_BASES) || m_base < 0)) @@ -658,6 +658,9 @@ namespace illumina { namespace interop { namespace model { namespace plot const id_t swath_beg = static_cast(ALL_IDS); const id_t swath_end = !supports_swath(plot_type) ? static_cast(ALL_IDS) : static_cast(info.flowcell().swath_count()+1); + const id_t section_beg = static_cast(ALL_IDS); + const id_t section_end = !supports_section(plot_type, info) ? static_cast(ALL_IDS) : + static_cast(info.flowcell().total_number_of_sections()+1); const id_t tile_beg = static_cast(ALL_IDS); const id_t tile_end = !supports_tile(plot_type) ? static_cast(ALL_IDS) : static_cast(info.flowcell().tile_count()+1); @@ -670,12 +673,25 @@ namespace illumina { namespace interop { namespace model { namespace plot new util::indirect_range_iterator(m_read, read_beg, read_end, !keep_state), new util::indirect_range_iterator(m_cycle, cycle_beg, cycle_end, !keep_state), new util::indirect_range_iterator(m_swath, swath_beg, swath_end, !keep_state), + new util::indirect_range_iterator(m_section, section_beg, section_end, !keep_state), new util::indirect_range_iterator(m_tile_number, tile_beg, tile_end, !keep_state) }; return util::chain_range_iterator(iterators); } public: + + /** Test if the combination of the plot and run info support filtering by section + * + * param plot_type type of plot + * @param info run info + * @return true if metric supports filtering by swath + */ + bool supports_section(const constants::plot_types /*plot_type*/, const model::run::info& info)const + { + if(info.flowcell().naming_method() != constants::FiveDigit) return false; + return false; + } /** Test if plot supports filtering by swath * * param plot_type type of plot @@ -830,6 +846,7 @@ namespace illumina { namespace interop { namespace model { namespace plot if(!options.all_reads()) out << "Read_" << options.m_read << "_"; if(!options.all_cycles()) out << "Cycle_" << options.m_cycle << "_"; if(!options.all_swaths()) out << "Swath_" << options.m_swath << "_"; + if(!options.all_sections()) out << "Section_" << options.m_section << "_"; if(!options.all_tile_numbers()) out << "Tile_" << options.m_tile_number << "_"; out << "_"; return out; diff --git a/interop/model/run/flowcell_layout.h b/interop/model/run/flowcell_layout.h index e1f4e7ba3..b861f4923 100644 --- a/interop/model/run/flowcell_layout.h +++ b/interop/model/run/flowcell_layout.h @@ -34,8 +34,8 @@ namespace illumina { namespace interop { namespace model { namespace run * @param surface_count number of surfaces * @param swath_count number of swaths * @param tile_count number of tiles - * @param sections_per_lane number of sections per lane - * @param lanes_per_section number of lanes per section + * @param sections_per_lane number of sections per lane (number of cameras per lane) + * @param lanes_per_section number of lanes per section (number of lanes a camera covers) * @param tiles vector of tile names * @param naming_method tile naming method * @param barcode flowcell barcode @@ -94,6 +94,7 @@ namespace illumina { namespace interop { namespace model { namespace run /** Get number of sections per lane * + * @note number of cameras per lane * @return number of sections per lane */ uint_t sections_per_lane() const @@ -101,11 +102,23 @@ namespace illumina { namespace interop { namespace model { namespace run /** Get number of lanes per section * + * @note number of lanes a camera covers * @return number of lanes per section */ uint_t lanes_per_section() const { return m_lanes_per_section; } + /** Get total number of sections + * + * @return total number of sections + */ + uint_t total_number_of_sections() const + { + if(m_naming_method == constants::FiveDigit) + return m_lane_count/m_lanes_per_section * m_sections_per_lane; + return 0; + } + /** Get the tile naming convention * * @return tile naming convention @@ -179,6 +192,18 @@ namespace illumina { namespace interop { namespace model { namespace run */ void tile_count(const uint_t tile_count) { m_tile_count = tile_count; } + /** Set number of sections per lane + * + * @param count number of sections per lane + */ + void sections_per_lane(const uint_t count) + { m_sections_per_lane = count; } + /** Set number of lanes per section + * + * @param count number of lanes per section + */ + void lanes_per_section(const uint_t count) + { m_lanes_per_section = count; } private: tile_naming_method_t m_naming_method; diff --git a/src/ext/csharp/CMakeLists.txt b/src/ext/csharp/CMakeLists.txt index 9bf69f183..6b4ddc925 100644 --- a/src/ext/csharp/CMakeLists.txt +++ b/src/ext/csharp/CMakeLists.txt @@ -135,6 +135,11 @@ if(BUILD_NUMBER) else() set(INTEROP_VERSION_STR ${VERSION_SHORT}) endif() +if("${CMAKE_HOST_SYSTEM}" MATCHES ".*Windows.*") + set(SHARED_LIB_SEARCH_STR "\\*${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(SHARED_LIB_SEARCH_STR "/*${CMAKE_SHARED_LIBRARY_SUFFIX}") +endif() # This creates the nuspec file in the proper directory add_custom_target(nuspec COMMENT "Creating package.nuspec" @@ -148,7 +153,7 @@ add_custom_target(nuspec -DINTEROP_VERSION=${INTEROP_VERSION_STR} -DPLATFORM=${NUGET_SYS_ID} -DCONFIG="${CONFIG_DIR}" - -DSHARED_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}" + -DSHARED_LIB_SEARCH="${SHARED_LIB_SEARCH_STR}" -DCONFIG_INPUT_FILE=${CMAKE_SOURCE_DIR}/cmake/package.nuspec.in -DCONFIG_OUTPUT_FILE=$/../package.nuspec -P ${CMAKE_SOURCE_DIR}/cmake/ConfigureFile.cmake diff --git a/src/interop/model/run/info.cpp b/src/interop/model/run/info.cpp index 82a65fee3..84c7eb2c9 100644 --- a/src/interop/model/run/info.cpp +++ b/src/interop/model/run/info.cpp @@ -226,7 +226,7 @@ namespace illumina { namespace interop { namespace model { namespace run if(surface > m_flowcell.surface_count()) INTEROP_THROW(invalid_run_info_exception, "Surface number exceeds number of surfaces"); const ::uint32_t section = logic::metric::section(tile, m_flowcell.naming_method()); - if(section > m_flowcell.sections_per_lane() * m_flowcell.lanes_per_section()) + if(m_flowcell.m_naming_method == constants::FiveDigit && section > m_flowcell.total_number_of_sections()) INTEROP_THROW(invalid_run_info_exception, "Section number exceeds number of sections"); } diff --git a/src/tests/interop/logic/inc/metric_filter_iterator.h b/src/tests/interop/logic/inc/metric_filter_iterator.h index f168c07c9..39b529a3a 100644 --- a/src/tests/interop/logic/inc/metric_filter_iterator.h +++ b/src/tests/interop/logic/inc/metric_filter_iterator.h @@ -172,9 +172,11 @@ namespace illumina { namespace interop { namespace unittest // Reduce the number of filters! model::run::flowcell_layout layout = run_info.flowcell(); - layout.lane_count(1); + layout.lane_count(1); // TODO: increase to test filtering? layout.swath_count(1); layout.tile_count(1); + layout.sections_per_lane(1); + layout.lanes_per_section(1); m_run_info.flowcell(layout); std::string channels[] = {"Red", "Green"}; m_run_info.channels(util::to_vector(channels)); From 215d7b33e8eac36c42d9c615c0d8ede330bd4f91 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Tue, 6 Dec 2016 15:23:02 -0800 Subject: [PATCH 3/4] IPA-4674: Add support for absolute naming convention --- docs/src/changes.md | 2 ++ interop/logic/metric/tile_metric.h | 8 +++++--- interop/model/metric_base/base_metric.h | 6 ++++-- tools/regression_test.ps1 | 2 ++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/src/changes.md b/docs/src/changes.md index c1de3f916..6905b5289 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -4,6 +4,8 @@ Date | Description ---------- | ----------- +2016-12-08 | Added support for absolute tile naming +2016-12-08 | Added ability to do section filtering 2016-12-05 | Added regression tests for imaging 2016-12-05 | Added unit tests for format coverage 2016-11-17 | Improved performance of InterOp calculations for C++11 enabled version diff --git a/interop/logic/metric/tile_metric.h b/interop/logic/metric/tile_metric.h index d52d7e1f0..d45b93a06 100644 --- a/interop/logic/metric/tile_metric.h +++ b/interop/logic/metric/tile_metric.h @@ -49,6 +49,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { if( metric.tile() > 9999) return constants::FiveDigit; if( metric.tile() > 999) return constants::FourDigit; + if( metric.tile() <= 99) return constants::Absolute; return constants::UnknownTileNamingMethod; } /** Determine the tile naming method from a metric set @@ -92,7 +93,8 @@ namespace illumina { namespace interop { namespace logic { namespace metric inline ::uint32_t surface(const ::uint32_t tile_id, const constants::tile_naming_method method) { if(method == constants::FiveDigit) return (tile_id / 10000); - return tile_id / 1000; + if(method == constants::FourDigit) return tile_id / 1000; + return 1; } /** Swath number * @@ -104,7 +106,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { if(method == constants::FiveDigit) return (tile_id / 1000) % 10; if(method == constants::FourDigit) return (tile_id / 100) % 10; - return 0; + return 1; } /** Determine the physical location (in terms of its column number) of the tile on the flow cell * @@ -119,7 +121,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric const ::uint32_t swath_count, const bool all_surfaces) { - if(!(method == constants::FiveDigit||method == constants::FourDigit)) return 0; + if(!(method == constants::FiveDigit || method == constants::FourDigit)) return 0; ::uint32_t col = swath(tile_id, method); if(all_surfaces && surface(tile_id, method)==2) col += swath_count; return col-1; diff --git a/interop/model/metric_base/base_metric.h b/interop/model/metric_base/base_metric.h index 0fd6f2bc0..7e00bd018 100644 --- a/interop/model/metric_base/base_metric.h +++ b/interop/model/metric_base/base_metric.h @@ -269,7 +269,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { if (method == constants::FiveDigit) return (m_tile / 10000); - return m_tile / 1000; + if (method == constants::FourDigit) + return m_tile / 1000; + return 1; } /** Swath number @@ -284,7 +286,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base return (m_tile / 1000) % 10; if (method == constants::FourDigit) return (m_tile / 100) % 10; - return 0; + return 1; } /** Index of the physical location of tile within the flowcell diff --git a/tools/regression_test.ps1 b/tools/regression_test.ps1 index 712bd89bf..a7f5a60bf 100644 --- a/tools/regression_test.ps1 +++ b/tools/regression_test.ps1 @@ -54,6 +54,8 @@ if ($test_code -ne 0) exit $test_code } +$baseline_path = $baseline_path + "_master" + $datasets = Get-ChildItem $data_path | foreach {$_.fullname} if($rebaseline) { From 873a40203b5a06ad44867d8adb1a842cc8e7c0d8 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Thu, 8 Dec 2016 10:01:00 -0800 Subject: [PATCH 4/4] IPA-5883: Fix possible memory issue in MSVC12 --- README.md | 1 + docs/src/changes.md | 1 + docs/src/index.md | 2 ++ interop/model/plot/heatmap_data.h | 5 +++-- interop/util/lexical_cast.h | 8 ++------ src/ext/swig/arrays/arrays_csharp_impl.i | 2 +- src/ext/swig/exceptions/exceptions_csharp.i | 9 +++++---- src/ext/swig/metrics.i | 2 ++ src/ext/swig/run.i | 2 ++ 9 files changed, 19 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2292d934d..5566974a7 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ There are several known limitations to the current library: 5. If both Visual Studio and Mono are installed, the build script will only use Visual Studio for C# 6. We do not support 32-bit builds 7. MinGW W64 4.9.x and prior will not link properly + 8. Visual Studio 12 2013 is not supported for the C# Bindings (Results in heap corruption) SAV Analysis Tab ---------------- diff --git a/docs/src/changes.md b/docs/src/changes.md index 6905b5289..b352fcc1c 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -4,6 +4,7 @@ Date | Description ---------- | ----------- +2016-12-08 | Added admonition to avoid using MSVC 12 (2013) with C# bindings 2016-12-08 | Added support for absolute tile naming 2016-12-08 | Added ability to do section filtering 2016-12-05 | Added regression tests for imaging diff --git a/docs/src/index.md b/docs/src/index.md index 6586d0c90..9b95aaba8 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -73,6 +73,8 @@ There are several known limitations to the current library: 4. We do not support Mono on Windows 5. If both Visual Studio and Mono are installed, the build script will only use Visual Studio for C# 6. We do not support 32-bit builds + 7. MinGW W64 4.9.x and prior will not link properly + 8. Visual Studio 12 2013 is not supported for the C# Bindings (Results in heap corruption) SAV Analysis Tab ---------------- diff --git a/interop/model/plot/heatmap_data.h b/interop/model/plot/heatmap_data.h index 461619e39..d6982cef2 100644 --- a/interop/model/plot/heatmap_data.h +++ b/interop/model/plot/heatmap_data.h @@ -227,8 +227,9 @@ namespace illumina { namespace interop { namespace model { namespace plot { in >> static_cast(data); std::string tmp; - const size_t col_count = io::table::read_value(in, tmp); - const size_t row_count = io::table::read_value(in, tmp); + size_t col_count, row_count; + io::table::read_value(in, col_count, tmp); + io::table::read_value(in, row_count, tmp); data.resize(row_count, col_count); io::table::read_csv(in, data.m_data, data.m_data+data.length()); return in; diff --git a/interop/util/lexical_cast.h b/interop/util/lexical_cast.h index 035a65968..f0119effc 100644 --- a/interop/util/lexical_cast.h +++ b/interop/util/lexical_cast.h @@ -78,12 +78,8 @@ namespace illumina { namespace interop { namespace util } static bool is_string_nan(const std::string &str) { - if (str.length() == 3 && ::tolower(str[0]) =='n' && ::tolower(str[1]) == 'a' && ::tolower(str[2]) == 'n') - { - return true; - } - if (str.length() == 4 && str[0] =='-' && ::tolower(str[1]) =='n' && ::tolower(str[2]) == 'a' && - ::tolower(str[3]) == 'n') + const size_t n = str.length(); + if(n >= 3 && ::tolower(str[n-3]) =='n' && ::tolower(str[n-2]) == 'a' && ::tolower(str[n-1]) == 'n') { return true; } diff --git a/src/ext/swig/arrays/arrays_csharp_impl.i b/src/ext/swig/arrays/arrays_csharp_impl.i index 1cfd878df..57695f563 100644 --- a/src/ext/swig/arrays/arrays_csharp_impl.i +++ b/src/ext/swig/arrays/arrays_csharp_impl.i @@ -124,7 +124,7 @@ CSHARP_ARRAYS_FIXED2(float, float) %apply float FIXED[] {float *} -%apply byte {unsigned char}; +//%apply byte {unsigned char}; diff --git a/src/ext/swig/exceptions/exceptions_csharp.i b/src/ext/swig/exceptions/exceptions_csharp.i index 3565f1996..d0118783e 100644 --- a/src/ext/swig/exceptions/exceptions_csharp.i +++ b/src/ext/swig/exceptions/exceptions_csharp.i @@ -16,7 +16,7 @@ // Note that SWIG detects any method calls named starting with // SWIG_CSharpSetPendingException for warning 845 - static void SWIG_CSharpSetPendingExceptionCustom_##EXCEPTION_CPLUS_PLUS##_##ENTRY_POINT(const char *msg) { + static void SWIGUNUSED SWIG_CSharpSetPendingExceptionCustom_##EXCEPTION_CPLUS_PLUS##_##ENTRY_POINT(const char *msg) { customExceptionCallback_##EXCEPTION_CPLUS_PLUS(msg); } %} @@ -28,12 +28,12 @@ static CustomExceptionDelegate customDelegate = new CustomExceptionDelegate(SetPendingCustomException); - [System.Runtime.InteropServices.DllImport("$dllimport", EntryPoint=#ENTRY_POINT)] + [global::System.Runtime.InteropServices.DllImport("$dllimport", EntryPoint=#ENTRY_POINT)] public static extern void ENTRY_POINT(CustomExceptionDelegate customCallback); static void SetPendingCustomException(string message) { - SWIGPendingException.Set(new Illumina.InterOp.Run.EXCEPTION_CSHARP(message)); + SWIGPendingException.Set(new Illumina.InterOp.Run.EXCEPTION_CSHARP(message, SWIGPendingException.Retrieve())); } static CustomExceptionHelper_##EXCEPTION_CPLUS_PLUS() { @@ -57,12 +57,13 @@ WRAP_EXCEPTION_IMPORT1(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_CSHARP, ENTRY_ %ignore EXCEPTION_CPLUS_PLUS(const std::string &mesg); %typemap(csinterfaces) NAMESPACE EXCEPTION_CPLUS_PLUS "" -%typemap(csbase, replace="1") NAMESPACE EXCEPTION_CPLUS_PLUS "System.ApplicationException" +%typemap(csbase, replace="1") NAMESPACE EXCEPTION_CPLUS_PLUS "global::System.ApplicationException" %typemap(csbody) NAMESPACE EXCEPTION_CPLUS_PLUS "" %typemap(csdestruct) NAMESPACE EXCEPTION_CPLUS_PLUS "" %typemap(csfinalize) NAMESPACE EXCEPTION_CPLUS_PLUS "" %typemap(cscode) NAMESPACE EXCEPTION_CPLUS_PLUS %{ public $csclassname(string mesg) : base(mesg){} + public $csclassname(string mesg, global::System.Exception inner) : base(mesg, inner){} %} %enddef diff --git a/src/ext/swig/metrics.i b/src/ext/swig/metrics.i index 59ea89e49..67e82d9a5 100644 --- a/src/ext/swig/metrics.i +++ b/src/ext/swig/metrics.i @@ -70,6 +70,7 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) using namespace illumina::interop::model::metrics; namespace metric_base = illumina::interop::model::metric_base; + %ignore illumina::interop::model::metric_base::metric_set::at(const size_t)const; %ignore illumina::interop::model::metric_base::metric_set::populate_tile_numbers_for_lane; %ignore illumina::interop::model::metric_base::metric_set::populate_tile_numbers_for_lane_surface; %ignore illumina::interop::model::metric_base::metric_set::offset_map; @@ -135,6 +136,7 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) %template(read_metric_vector) std::vector< illumina::interop::model::metrics::read_metric >; %template(q_score_bin_vector) std::vector< illumina::interop::model::metrics::q_score_bin >; + %template(metric_type_name_pair) std::pair< illumina::interop::constants::metric_type, std::string>; %template(metric_type_description) illumina::interop::constants::enum_description< illumina::interop::constants::metric_type>; %template(metric_type_description_vector) std::vector< illumina::interop::constants::enum_description< illumina::interop::constants::metric_type> >; diff --git a/src/ext/swig/run.i b/src/ext/swig/run.i index 0fb1970c6..4f9499652 100644 --- a/src/ext/swig/run.i +++ b/src/ext/swig/run.i @@ -15,6 +15,8 @@ } %} +%ignore operator enum_t; + %{ #include "interop/interop.h" #include "interop/model/run/cycle_range.h"