diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c0692e7a..439ddc1ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ set(INTEROP_DL_LIB interop_fpic_lib) option(ENABLE_BACKWARDS_COMPATIBILITY "Compile code for c++98" ON) option(ENABLE_DOCS "Build documentation with Doxygen" ON) option(ENABLE_SWIG "Build third-party language bindings, e.g. C#" ON) +option(ENABLE_CSHARP "Build C# language bindings" ON) option(ENABLE_TEST "Build unit tests (depends on Boost)" ON) option(ENABLE_APPS "Build command line programs" ON) option(ENABLE_EXAMPLES "Build example programs" ON) @@ -75,6 +76,7 @@ if(ENABLE_DOCS) add_subdirectory("docs") endif() install(DIRECTORY interop DESTINATION include) +install(FILES README.md docs/src/changes.md DESTINATION .) find_package(Git) if(GIT_FOUND) diff --git a/cmake/InternalUtils.cmake b/cmake/InternalUtils.cmake index dcc1e8192..3068bf3de 100644 --- a/cmake/InternalUtils.cmake +++ b/cmake/InternalUtils.cmake @@ -42,8 +42,10 @@ macro(interop_config_compiler_and_linker) include(CheckIsNaN) - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR CMAKE_COMPILER_IS_GNUCC) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" OR CMAKE_COMPILER_IS_GNUCC OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(COMPILER_IS_GNUCC_OR_CLANG ON) + else() + set(COMPILER_IS_GNUCC_OR_CLANG OFF) endif() if(CMAKE_SIZEOF_VOID_P EQUAL "4") @@ -72,7 +74,7 @@ macro(interop_config_compiler_and_linker) if(IS_INT64_LONG) add_definitions(-DHAVE_LONG64) endif() - + check_type_size(size_t SIZE_T) if(MINGW) set(CXX11_FLAG_ "-std=gnu++11") else() @@ -101,12 +103,17 @@ macro(interop_config_compiler_and_linker) message(FATAL_ERROR "Unsupported compiler") endif() endif() - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" OR CMAKE_COMPILER_IS_GNUCC) + if(COMPILER_IS_GNUCC_OR_CLANG) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -Wno-c++0x-compat -Wno-error=c++0x-compat -Wextra") + if(WIN32 AND MINGW) + # Add big object support to Windows Compilers + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj") + endif() elseif(MSVC) # Visual Studio Complains about not being able to create an assignment operator and copy constructor # -wd4511 and -wd4512 disable these pointless warnings - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX -wd4511 -wd4512") + # Add big object support to Windows Compilers + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX -wd4511 -wd4512 /bigobj") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") if(WIN32) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") @@ -115,7 +122,7 @@ macro(interop_config_compiler_and_linker) endif() endif() if(NOT ENABLE_BACKWARDS_COMPATIBILITY) - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" OR CMAKE_COMPILER_IS_GNUCC) + if(COMPILER_IS_GNUCC_OR_CLANG) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_CX11_FLAG}") endif() elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") diff --git a/cmake/Modules/FindGMock.cmake b/cmake/Modules/FindGMock.cmake index fe1812bb0..d3a3f5e90 100644 --- a/cmake/Modules/FindGMock.cmake +++ b/cmake/Modules/FindGMock.cmake @@ -31,11 +31,24 @@ find_library(GMOCK_LIBRARY PATH_SUFFIXES . lib lib64 ) find_library(GMOCK_MAIN_LIBRARY - NAMES gmock_main$ + NAMES gmock_main HINTS ${PC_GMOCK_INCLUDEDIR} ${PC_GMOCK_INCLUDE_DIRS} ${GMOCK_ROOT} $ENV{GMOCK_DIR} PATH_SUFFIXES . lib lib64 ) + +find_library(GMOCK_LIBRARY_DEBUG + NAMES gmock-d + HINTS ${PC_GMOCK_INCLUDEDIR} ${PC_GMOCK_INCLUDE_DIRS} ${GMOCK_ROOT} $ENV{GMOCK_DIR} + PATH_SUFFIXES . lib lib64 +) +find_library(GMOCK_MAIN_LIBRARY_DEBUG + NAMES gmock_main-d + HINTS ${PC_GMOCK_INCLUDEDIR} ${PC_GMOCK_INCLUDE_DIRS} ${GMOCK_ROOT} $ENV{GMOCK_DIR} + PATH_SUFFIXES . lib lib64 +) + + if(NOT GMOCK_INCLUDE_DIR OR NOT GMOCK_LIBRARY OR NOT GMOCK_MAIN_LIBRARY) if(NOT GMOCK_INCLUDE_DIR) @@ -82,7 +95,20 @@ if(NOT GMOCK_INCLUDE_DIR OR NOT GMOCK_LIBRARY OR NOT GMOCK_MAIN_LIBRARY) set(GMOCK_TARGET googlemock) endif() - +if(GMOCK_LIBRARY AND GMOCK_LIBRARY_DEBUG) + get_filename_component(GMOCK_RELEASE_EXT ${GMOCK_LIBRARY} EXT) + get_filename_component(GMOCK_DEBUG_EXT ${GMOCK_LIBRARY_DEBUG} EXT) + if(GMOCK_RELEASE_EXT STREQUAL GMOCK_DEBUG_EXT) + set(GMOCK_LIBRARY optimized ${GMOCK_LIBRARY} debug ${GMOCK_LIBRARY_DEBUG}) + endif() +endif() +if(GMOCK_MAIN_LIBRARY AND GMOCK_MAIN_LIBRARY_DEBUG) + get_filename_component(GMOCK_RELEASE_EXT ${GMOCK_MAIN_LIBRARY} EXT) + get_filename_component(GMOCK_DEBUG_EXT ${GMOCK_MAIN_LIBRARY_DEBUG} EXT) + if(GMOCK_RELEASE_EXT STREQUAL GMOCK_DEBUG_EXT) + set(GMOCK_MAIN_LIBRARY optimized ${GMOCK_MAIN_LIBRARY} debug ${GMOCK_MAIN_LIBRARY_DEBUG}) + endif() +endif() set(GMOCK_LIBRARIES ${GMOCK_LIBRARY} ) set(GMOCK_INCLUDE_DIRS ${GMOCK_INCLUDE_DIR} ) diff --git a/cmake/Modules/FindGTest.cmake b/cmake/Modules/FindGTest.cmake index 6ebf7b46f..bfff20c73 100644 --- a/cmake/Modules/FindGTest.cmake +++ b/cmake/Modules/FindGTest.cmake @@ -31,12 +31,40 @@ find_library(GTEST_LIBRARY PATH_SUFFIXES . lib lib64 ) find_library(GTEST_MAIN_LIBRARY - NAMES gtest_main$ + NAMES gtest_main HINTS ${PC_GTEST_INCLUDEDIR} ${PC_GTEST_INCLUDE_DIRS} ${GTEST_ROOT} $ENV{GTEST_DIR} PATH_SUFFIXES . lib lib64 ) -if(NOT GTEST_INCLUDE_DIR OR NOT GTEST_LIBRARY OR NOT GTEST_MAIN_LIBRARY) +find_library(GTEST_LIBRARY_DEBUG + NAMES gtest-d + HINTS ${PC_GTEST_INCLUDEDIR} ${PC_GTEST_INCLUDE_DIRS} ${GTEST_ROOT} $ENV{GTEST_DIR} + PATH_SUFFIXES . lib lib64 + ) +find_library(GTEST_MAIN_LIBRARY_DEBUG + NAMES gtest_main-d + HINTS ${PC_GTEST_INCLUDEDIR} ${PC_GTEST_INCLUDE_DIRS} ${GTEST_ROOT} $ENV{GTEST_DIR} + PATH_SUFFIXES . lib lib64 + ) + +find_path(GMOCK_INCLUDE_DIR_TEST + NAMES gmock/gmock.h + HINTS ${PC_GMOCK_INCLUDEDIR} ${PC_GMOCK_INCLUDE_DIRS} ${GMOCK_ROOT} $ENV{GMOCK_DIR} + PATH_SUFFIXES . include + ) + +find_library(GMOCK_LIBRARY_TEST + NAMES gmock + HINTS ${PC_GMOCK_INCLUDEDIR} ${PC_GMOCK_INCLUDE_DIRS} ${GMOCK_ROOT} $ENV{GMOCK_DIR} + PATH_SUFFIXES . lib lib64 + ) +find_library(GMOCK_MAIN_LIBRARY_TEST + NAMES gmock_main + HINTS ${PC_GMOCK_INCLUDEDIR} ${PC_GMOCK_INCLUDE_DIRS} ${GMOCK_ROOT} $ENV{GMOCK_DIR} + PATH_SUFFIXES . lib lib64 + ) + +if(NOT GTEST_INCLUDE_DIR OR NOT GTEST_LIBRARY OR NOT GTEST_MAIN_LIBRARY OR NOT GMOCK_INCLUDE_DIR_TEST OR NOT GMOCK_LIBRARY_TEST OR NOT GMOCK_MAIN_LIBRARY_TEST) if(NOT GTEST_INCLUDE_DIR) message(STATUS "GTest include directory not found") @@ -82,7 +110,20 @@ if(NOT GTEST_INCLUDE_DIR OR NOT GTEST_LIBRARY OR NOT GTEST_MAIN_LIBRARY) set(GTEST_TARGET gtest) endif() - +if(GTEST_LIBRARY AND GTEST_LIBRARY_DEBUG) + get_filename_component(GTEST_RELEASE_EXT ${GTEST_LIBRARY} EXT) + get_filename_component(GTEST_DEBUG_EXT ${GTEST_LIBRARY_DEBUG} EXT) + if(GTEST_RELEASE_EXT STREQUAL GTEST_DEBUG_EXT) + set(GTEST_LIBRARY optimized ${GTEST_LIBRARY} debug ${GTEST_LIBRARY_DEBUG}) + endif() +endif() +if(GTEST_MAIN_LIBRARY AND GTEST_MAIN_LIBRARY_DEBUG) + get_filename_component(GTEST_RELEASE_EXT ${GTEST_MAIN_LIBRARY} EXT) + get_filename_component(GTEST_DEBUG_EXT ${GTEST_MAIN_LIBRARY_DEBUG} EXT) + if(GTEST_RELEASE_EXT STREQUAL GTEST_DEBUG_EXT) + set(GTEST_MAIN_LIBRARY optimized ${GTEST_MAIN_LIBRARY} debug ${GTEST_MAIN_LIBRARY_DEBUG}) + endif() +endif() set(GTEST_LIBRARIES ${GTEST_LIBRARY} ) set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIR} ) diff --git a/cmake/Modules/UseCSharp.cmake b/cmake/Modules/UseCSharp.cmake index bd04ca0ec..d4253dafd 100644 --- a/cmake/Modules/UseCSharp.cmake +++ b/cmake/Modules/UseCSharp.cmake @@ -92,7 +92,6 @@ macro( CSHARP_ADD_PROJECT type name ) #endif () #list( SORT sources_dep ) - set(CSHARP_OUTPUT_DIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CSHARP_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) string (REPLACE ";" "," source_list "${sources}") @@ -101,17 +100,17 @@ macro( CSHARP_ADD_PROJECT type name ) # Add custom target and command MESSAGE( STATUS "Adding C# ${type} ${name}: '${CSHARP_COMPILER} /t:${type} /out:${name}.${output} /platform:${CSHARP_PLATFORM} ${CSHARP_SDK} ${refs} ${sources}'" ) add_custom_command( - COMMENT "Compiling C# ${type} ${name}: '${CSHARP_COMPILER} /unsafe /t:${type} /out:${CSHARP_OUTPUT_DIR}/${name}.${output} /platform:${CSHARP_PLATFORM} ${CSHARP_SDK} ${refs} ${sources}'" - OUTPUT ${CMAKE_BINARY_DIR}/${name}.${output} - COMMAND ${CMAKE_COMMAND} -DFILES_TO_COPY="${source_list}" -DDESTINATION_DIR="${CMAKE_BINARY_DIR}" -P "${CMAKE_SOURCE_DIR}/cmake/CopyListOfFiles.cmake" + COMMENT "Compiling C# ${type} ${name}: '${CSHARP_COMPILER} /unsafe /t:${type} /out:${CMAKE_CURRENT_BINARY_DIR}/${name}.${output} /platform:${CSHARP_PLATFORM} ${CSHARP_SDK} ${refs} ${sources}'" + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${name}.${output} + COMMAND ${CMAKE_COMMAND} -DFILES_TO_COPY="${source_list}" -DDESTINATION_DIR="${CMAKE_CURRENT_BINARY_DIR}" -P "${CMAKE_SOURCE_DIR}/cmake/CopyListOfFiles.cmake" COMMAND ${CSHARP_COMPILER} ARGS /t:${type} /out:${CSHARP_OUTPUT_DIR}/${name}.${output} /unsafe /platform:${CSHARP_PLATFORM} ${CSHARP_SDK} ${refs} ${sources} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${sources_dep} ) add_custom_target( ${name} ALL - DEPENDS ${CMAKE_BINARY_DIR}/${name}.${output} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${name}.${output} SOURCES ${sources_dep} ) endmacro( CSHARP_ADD_PROJECT ) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 71942acff..9d47ceccf 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -416,7 +416,7 @@ LOOKUP_CACHE_SIZE = 0 # normally produced when WARNINGS is set to YES. # The default value is: NO. -EXTRACT_ALL = NO +EXTRACT_ALL = YES # If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will # be included in the documentation. @@ -654,7 +654,7 @@ SHOW_USED_FILES = YES # (if specified). # The default value is: YES. -SHOW_FILES = NO +SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces # page. This will remove the Namespaces entry from the Quick Index and from the @@ -716,6 +716,9 @@ QUIET = NO WARNINGS = YES +# Treat warnings as errors +#WARN_AS_ERROR = YES + # If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. @@ -737,7 +740,7 @@ WARN_IF_DOC_ERROR = YES # parameter documentation, but not about the absence of documentation. # The default value is: NO. -WARN_NO_PARAMDOC = NO +WARN_NO_PARAMDOC = YES # The WARN_FORMAT tag determines the format of the warning messages that doxygen # can produce. The string should contain the $file, $line, and $text tags, which diff --git a/docs/src/changes.md b/docs/src/changes.md index 0307e4da1..729e9240f 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -1,10 +1,32 @@ # Changes {#changes} -## Trunk - -Commit | Description -------- | ----------- +## v1.0.9 + +Date | Description +---------- | ----------- +2016-08-26 | Refactor CSharp code into additional library +2016-08-26 | IPA-5028: Add RunInfo and InterOp validation +2016-08-19 | Fix sorting in imaging table +2016-08-18 | Cleanup code +2016-08-17 | Fix the read ID +2016-08-16 | IPA-5009: Add sorting to imaging table +2016-08-16 | IPA-5007: Fix combo boxes on analysis tab +2016-08-16 | Fix msvc warning and some script differences +2016-08-15 | Add clear function to run metrics +2016-08-15 | IPA-4799: Create Internal Coverity Build +2016-08-11 | Refactor unit tests to make more flexible for regression testing +2016-08-08 | IPA-4922: Move logic code from header to cpp +2016-08-05 | IPA-4827: Make run info parsing more robust +2016-08-03 | Refactor code and add test build script +2016-08-02 | Add big object support for unit tests +2016-08-02 | Clean buffers in the heatmap and flowcell +2016-08-02 | Update travis-ci to GCC-5 +2016-07-28 | Added float array backing to heat map +2016-07-25 | IPA-4752: Replace InterOp 1.x Code with InterOp 2.x Code - Imaging Tab +2016-07-22 | Fix camel case +2016-07-22 | IPA-4751: Integrated InterOp2.x with Analysis Tab +2016-07-22 | Github Issue 74: Compiler warning as error unsigned/signed comp 2016-07-21 | Bug fixes for flowcell chart and more recent versions of GCC 2016-07-17 | Added coverity online scan and fixed some issues found diff --git a/docs/src/performance.md b/docs/src/performance.md index 8086e694f..90f216120 100644 --- a/docs/src/performance.md +++ b/docs/src/performance.md @@ -63,6 +63,6 @@ Direct Call 1 | 38 Direct Call 2 | 45 Indirect Call | < 1 -The results show that everytime we make a call from the native language to C++, we pay a hefty price. In many applications -this is a price we are willing to pay. However, for those applications were performance is a concern, it pays to add -methods like `copy_focus`, which limit the number of times we have to jump between C# (or another language) and C++. \ No newline at end of file +The results show that everytime we make a call from the native language to C++, we pay a hefty price. In many applications, +this is a price we are willing to pay. However, for those applications where performance is a concern, it pays to add +methods like `copy_focus`, which limit the number of times we have to jump between C# (or another language) and C++. diff --git a/interop/constants/enums.h b/interop/constants/enums.h index 41f88335d..830108c95 100644 --- a/interop/constants/enums.h +++ b/interop/constants/enums.h @@ -24,6 +24,7 @@ INTEROP_TUPLE2(ChannelFeature, 0x10), \ INTEROP_TUPLE1(UnknownMetricFeature) + /** Enumeration of each metric type * * @note This macro requires the macro INTEROP_TUPLE3 to be defined before use @@ -34,6 +35,7 @@ INTEROP_TUPLE4(FWHM, "FWHM", Extraction, CycleFeature|ChannelFeature),\ /*INTEROP_TUPLE4(PercentBase, "% Base", CorrectedInt, CycleFeature|BaseFeature), */\ INTEROP_TUPLE4(BasePercent, "% Base", CorrectedInt, CycleFeature|BaseFeature),\ + INTEROP_TUPLE4(PercentNoCall, "% NoCall", CorrectedInt, CycleFeature),\ /*INTEROP_TUPLE4(PercentQ20, "% >=Q20", Q, CycleFeature),*/\ INTEROP_TUPLE4(Q20Percent, "% >=Q20", Q, CycleFeature),\ /*INTEROP_TUPLE4(PercentQ30, "% >=Q30", Q, CycleFeature),*/\ @@ -45,7 +47,7 @@ /*INTEROP_TUPLE4(Density, "Density (K/mm2)", Tile, TileFeature),*/ \ INTEROP_TUPLE4(ClustersPF, "Density PF", Tile, TileFeature),\ /*INTEROP_TUPLE4(DensityPF, "Density PF", Tile, TileFeature),*/\ - INTEROP_TUPLE4(ClusterCount, "Clusters", Tile, TileFeature),\ + INTEROP_TUPLE4(ClusterCount, "Cluster Count", Tile, TileFeature),\ /*INTEROP_TUPLE4(ClusterCount, "Cluster Count", Tile, TileFeature),*/\ INTEROP_TUPLE4(ClusterCountPF, "Clusters PF", Tile, TileFeature),\ INTEROP_TUPLE4(ErrorRate, "Error Rate", Error, CycleFeature),\ @@ -157,10 +159,10 @@ INTEROP_TUPLE1(InstrumentCount), \ INTEROP_TUPLE1(UnknownInstrument) -/** Enumeration of instrument types +/** Enumeration of metric types * * @note This macro requires the macro INTEROP_TUPLE1 to be defined before use - * @see illumina::interop::constants::instrument_type + * @see illumina::interop::constants::metric_base_type */ #define INTEROP_ENUM_METRIC_BASE_TYPES \ /** Tile base types are written out once for each tile */\ @@ -169,6 +171,8 @@ INTEROP_TUPLE1(BaseCycleType),\ /** Read base types are written out once for each tile and read */\ INTEROP_TUPLE1(BaseReadType),\ + /** Lane base types are written out once for each lane and cycle */\ + INTEROP_TUPLE1(BaseLaneType),\ INTEROP_TUPLE1(BaseMetricCount),\ INTEROP_TUPLE1(UnknownBaseType) @@ -201,6 +205,7 @@ INTEROP_TUPLE1(Shifted),\ INTEROP_TUPLE1(UnknownBarPlotOption) +/** Sentinel for an unknown enum type */ #define INTEROP_UNKNOWN 0xff /** This temp macro converts a enum/description pair into the an enum (first element of the pair) */ #define INTEROP_TUPLE4(X, Y, Z, A) X @@ -211,6 +216,8 @@ /** This temp macro converts an enum/value pair to an enum */ #define INTEROP_TUPLE2(X, V) X=V + + namespace illumina { namespace interop { namespace constants { /** Common codes for all enums */ diff --git a/interop/constants/typedefs.h b/interop/constants/typedefs.h index 6153a0fa7..90fc4851f 100644 --- a/interop/constants/typedefs.h +++ b/interop/constants/typedefs.h @@ -19,5 +19,7 @@ namespace illumina { namespace interop { namespace constants { typedef constant_type base_cycle_t; /** Define base type for read metrics */ typedef constant_type base_read_t; + /** Define base type for lane metrics */ + typedef constant_type base_lane_t; }}} diff --git a/interop/interop.h b/interop/interop.h index cdad69d53..bf9b60e38 100644 --- a/interop/interop.h +++ b/interop/interop.h @@ -18,7 +18,8 @@ #include "interop/logic/summary/index_summary.h" #include "interop/logic/summary/run_summary.h" -namespace illumina { namespace interop{ +namespace illumina { namespace interop +{ /** Get the version of the library * * @return library version diff --git a/interop/io/format/map_io.h b/interop/io/format/map_io.h index 1e2777faa..ca24f16bc 100644 --- a/interop/io/format/map_io.h +++ b/interop/io/format/map_io.h @@ -58,12 +58,12 @@ namespace illumina { namespace interop { namespace io * TODO: create more efficient buffered version * * @param in input stream - * @param vals destinatin array of values + * @param vals destination array of values * @param n number of values to read * @return number of bytes read from the stream */ template - std::streamsize stream_map(std::istream &in, ValueType &vals, size_t n) + std::streamsize stream_map(std::istream &in, ValueType &vals, const size_t n) { std::streamsize tot = 0; vals.resize(n); @@ -82,12 +82,37 @@ namespace illumina { namespace interop { namespace io * TODO: create more efficient buffered version * * @param in input stream - * @param vals destinatin array of values + * @param vals destination array of values + * @param offset starting index of values to copy to in vals + * @param n number of values to read + * @return number of bytes read from the stream + */ + template + std::streamsize stream_map(std::istream &in, ValueType &vals, const size_t offset, const size_t n) + { + std::streamsize tot = 0; + vals.resize(offset + n); + for (size_t i = 0; i < n; i++) + { + ReadType read_val; + read_binary(in, read_val); + vals[offset + i] = read_val; + tot += in.gcount(); + } + return tot; + } + + /** Read an array of values of type ReadType from the given input stream + * + * TODO: create more efficient buffered version + * + * @param in input stream + * @param vals destination array of values * @param n number of values in array * @return number of bytes read from the stream */ template - std::streamsize stream_map(std::istream &in, ValueType (&vals)[N], size_t n) + std::streamsize stream_map(std::istream &in, ValueType (&vals)[N], const size_t n) { std::streamsize tot = 0; for (size_t i = 0; i < n; i++) @@ -107,7 +132,7 @@ namespace illumina { namespace interop { namespace io * @return number of bytes written to the stream */ template - std::streamsize stream_map(std::ostream &out, ValueType val) + std::streamsize stream_map(std::ostream &out, const ValueType val) { write_binary(out, static_cast(val)); return out.tellp(); @@ -118,12 +143,12 @@ namespace illumina { namespace interop { namespace io * TODO: create more efficient buffered version * * @param out output stream - * @param vals destinatin array of values + * @param vals destination array of values * @param n number of values in array * @return number of bytes written to the stream */ template - std::streamsize stream_map(std::ostream &out, const ValueType &vals, size_t n) + std::streamsize stream_map(std::ostream &out, const ValueType &vals, const size_t n) { for (size_t i = 0; i < n; i++) { @@ -133,7 +158,28 @@ namespace illumina { namespace interop { namespace io return out.tellp(); } - /** Place older that does nothing + /** Write an array of values of type ReadType to the given output stream + * + * TODO: create more efficient buffered version + * + * @param out output stream + * @param vals destination array of values + * @param offset starting index of values to write to in vals + * @param n number of values in array + * @return number of bytes written to the stream + */ + template + std::streamsize stream_map(std::ostream &out, const ValueType &vals, const size_t offset, const size_t n) + { + for (size_t i = 0; i < n; i++) + { + WriteType write_val = vals[offset + i]; + write_binary(out, write_val); + } + return out.tellp(); + } + + /** Placeholder that does nothing */ template void map_resize(const std::vector &, size_t) @@ -146,7 +192,7 @@ namespace illumina { namespace interop { namespace io * @param n number of elements */ template - void map_resize(std::vector &layout, size_t n) + void map_resize(std::vector &layout, const size_t n) { layout.resize(n); } diff --git a/interop/io/format/metric_format.h b/interop/io/format/metric_format.h index 46409d906..ed944c8b7 100644 --- a/interop/io/format/metric_format.h +++ b/interop/io/format/metric_format.h @@ -72,20 +72,25 @@ namespace illumina { namespace interop { namespace io { if (in.fail()) INTEROP_THROW(incomplete_file_exception, "Insufficient header data read from the file"); + + //if we're not actually reading the record size from the stream (the stream position is the same before and after), + // then don't compare the layout size against the record size + const ::int64_t stream_position_pre_record_check = in.tellg(); const std::streamsize record_size = Layout::map_stream_record_size(in, static_cast(0)); + const ::int64_t stream_position_post_record_check = in.tellg(); Layout::map_stream_for_header(in, header); if (in.fail()) INTEROP_THROW(incomplete_file_exception, "Insufficient extended header data read from the file"); const std::streamsize layout_size = Layout::compute_size(header); - if (record_size != layout_size) + if (stream_position_pre_record_check != stream_position_post_record_check && record_size != layout_size) INTEROP_THROW(bad_format_exception, "Record size does not match layout size, record size: " << record_size << " != layout size: " << layout_size << " for " << Metric::prefix() << " " << Metric::suffix() << " v" << Layout::VERSION); - return record_size; + return layout_size; } /** Read a metric set from the given input stream diff --git a/interop/io/layout/base_metric.h b/interop/io/layout/base_metric.h index 25810502f..32806e8c1 100644 --- a/interop/io/layout/base_metric.h +++ b/interop/io/layout/base_metric.h @@ -9,7 +9,6 @@ * @note These classes are packed such that there is not padding. Their size reflects the accumulation of their member fields. * * @file - * * @date 7/30/15 * @version 1.0 * @copyright GNU Public License. diff --git a/interop/io/metric_file_stream.h b/interop/io/metric_file_stream.h index 408bcb728..a500ec2b3 100644 --- a/interop/io/metric_file_stream.h +++ b/interop/io/metric_file_stream.h @@ -11,7 +11,8 @@ #include "interop/util/exception.h" #include "interop/io/metric_stream.h" -namespace illumina { namespace interop { namespace io { +namespace illumina { namespace interop { namespace io +{ /** @defgroup file_io Reading/Writing Binary InterOp files * @@ -91,11 +92,31 @@ namespace illumina { namespace interop { namespace io { interop::io::incomplete_file_exception, model::index_out_of_bounds_exception) { - const std::string fileName = interop_filename(run_directory, use_out); - std::ifstream fin(fileName.c_str(), std::ios::binary); - if(!fin.good()) INTEROP_THROW(file_not_found_exception, "File not found: " << fileName); + const std::string file_name = interop_filename(run_directory, use_out); + std::ifstream fin(file_name.c_str(), std::ios::binary); + if(!fin.good()) INTEROP_THROW(file_not_found_exception, "File not found: " << file_name); read_metrics(fin, metrics); } + /** Check for the existence of the binary InterOp file into the given metric set + * + * @note The 'Out' suffix (parameter: use_out) is appended when we read the file. We excluded the Out in certain + * conditions when writing the file. + * + * @param run_directory file path to the run directory + * @param use_out use the copied version + */ + template + bool interop_exists(const std::string& run_directory, MetricSet&, const bool use_out=true) throw + (interop::io::file_not_found_exception, + interop::io::bad_format_exception, + interop::io::incomplete_file_exception, + model::index_out_of_bounds_exception) + { + const std::string file_name = interop_filename(run_directory, use_out); + std::ifstream fin(file_name.c_str(), std::ios::binary); + if(!fin.good()) return false; + return true; + } /** Write the metric set to a binary InterOp file * * @note The 'Out' suffix (parameter: use_out) is appended when we read the file. We excluded the Out in certain @@ -112,9 +133,10 @@ namespace illumina { namespace interop { namespace io { const bool use_out=true, const ::int16_t version=-1) throw(file_not_found_exception, bad_format_exception, incomplete_file_exception) { - const std::string fileName = interop_filename(run_directory, use_out); - std::ofstream fout(fileName.c_str(), std::ios::binary); - if(!fout.good())INTEROP_THROW(file_not_found_exception, "File not found: " << fileName); + if(metrics.empty())return; + const std::string file_name = interop_filename(run_directory, use_out); + std::ofstream fout(file_name.c_str(), std::ios::binary); + if(!fout.good())INTEROP_THROW(file_not_found_exception, "File not found: " << file_name); write_metrics(fout, metrics, version); } /** Write only the header to a binary InterOp file @@ -134,9 +156,9 @@ namespace illumina { namespace interop { namespace io { const bool use_out=true) throw(file_not_found_exception, bad_format_exception, incomplete_file_exception) { - const std::string fileName = interop_filename(run_directory, use_out); - std::ofstream fout(fileName.c_str(), std::ios::binary); - if(!fout.good())INTEROP_THROW(file_not_found_exception, "File not found: " << fileName); + const std::string file_name = interop_filename(run_directory, use_out); + std::ofstream fout(file_name.c_str(), std::ios::binary); + if(!fout.good())INTEROP_THROW(file_not_found_exception, "File not found: " << file_name); write_metric_header(fout, header, version); } /** @} */ diff --git a/interop/io/metric_stream.h b/interop/io/metric_stream.h index 2d883460c..aa23df19e 100644 --- a/interop/io/metric_stream.h +++ b/interop/io/metric_stream.h @@ -22,12 +22,12 @@ namespace illumina { namespace interop { namespace io { /** Append the InterOp directory to the run directory file path * - * @param runDirectory file path to the run directory + * @param run_directory file path to the run directory * @return file path to the InterOp directory */ - inline std::string interop_directory_name(const std::string &runDirectory) + inline std::string interop_directory_name(const std::string &run_directory) { - return io::combine(runDirectory, "InterOp"); + return io::combine(run_directory, "InterOp"); } /** Generate a file name from a run directory and the InterOp name @@ -37,14 +37,14 @@ namespace illumina { namespace interop { namespace io * * @param prefix prefix name of the interop file * @param suffix suffix name of the interop file - * @param useOut if true, append "Out" to the end of the filename + * @param use_out if true, append "Out" to the end of the filename * @return file path to the InterOp directory */ inline std::string interop_basename(const std::string &prefix, const std::string &suffix, - const bool useOut = true) + const bool use_out = true) { - return prefix + "Metrics" + suffix + ((useOut) ? ("Out.bin") : (".bin")); + return prefix + "Metrics" + suffix + ((use_out) ? ("Out.bin") : (".bin")); } /** Generate a file name from a run directory and the InterOp name @@ -52,22 +52,22 @@ namespace illumina { namespace interop { namespace io * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions * when writing the file. * - * @param runDirectory file path to the run directory + * @param run_directory file path to the run directory * @param prefix prefix name of the interop file * @param suffix suffix name of the interop file - * @param useOut if true, append "Out" to the end of the filename + * @param use_out if true, append "Out" to the end of the filename * @return file path to the InterOp directory */ - inline std::string interop_filename(const std::string &runDirectory, + inline std::string interop_filename(const std::string &run_directory, const std::string &prefix, const std::string &suffix, - const bool useOut = true) + const bool use_out = true) { - if (io::basename(runDirectory) == interop_basename(prefix, suffix, useOut)) - return runDirectory; - if (io::basename(runDirectory) == "InterOp") - return io::combine(runDirectory, interop_basename(prefix, suffix, useOut)); - return io::combine(interop_directory_name(runDirectory), interop_basename(prefix, suffix, useOut)); + if (io::basename(run_directory) == interop_basename(prefix, suffix, use_out)) + return run_directory; + if (io::basename(run_directory) == "InterOp") + return io::combine(run_directory, interop_basename(prefix, suffix, use_out)); + return io::combine(interop_directory_name(run_directory), interop_basename(prefix, suffix, use_out)); } /** Memory buffer for a stream @@ -93,14 +93,14 @@ namespace illumina { namespace interop { namespace io * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions * when writing the file. * - * @param runDirectory file path to the run directory - * @param useOut if true, append "Out" to the end of the filename + * @param run_directory file path to the run directory + * @param use_out if true, append "Out" to the end of the filename * @return file path to the InterOp directory */ template - std::string interop_filename(const std::string &runDirectory, bool useOut = true) + std::string interop_filename(const std::string &run_directory, bool use_out = true) { - return detail::interop_filename(runDirectory, MetricType::prefix(), MetricType::suffix(), useOut); + return detail::interop_filename(run_directory, MetricType::prefix(), MetricType::suffix(), use_out); } /** Generate a file name from a run directory and the metric type @@ -108,13 +108,13 @@ namespace illumina { namespace interop { namespace io * @note The 'Out' suffix is appended when we read the file. We excluded the Out in certain conditions * when writing the file. * - * @param useOut if true, append "Out" to the end of the filename + * @param use_out if true, append "Out" to the end of the filename * @return file path to the InterOp directory */ template - std::string interop_basename(bool useOut = true) + std::string interop_basename(bool use_out = true) { - return detail::interop_basename(MetricType::prefix(), MetricType::suffix(), useOut); + return detail::interop_basename(MetricType::prefix(), MetricType::suffix(), use_out); } /** Read the binary InterOp file into the given metric set @@ -309,7 +309,9 @@ namespace illumina { namespace interop { namespace io if (version < 0) version = metrics.version(); if (format_map.find(version) == format_map.end()) INTEROP_THROW(bad_format_exception, "No format found to write file with version: " << - version << " of " << format_map.size()); + version << " of " << format_map.size() + << " for " << metric_type::prefix() << "" << metric_type::suffix() + << " with " << metrics.size() << " metrics"); INTEROP_ASSERT(format_map[version]); format_map[version]->write_metric_header(out, metrics); diff --git a/interop/io/plot/gnuplot.h b/interop/io/plot/gnuplot.h index 5a68f2900..27d32bd92 100644 --- a/interop/io/plot/gnuplot.h +++ b/interop/io/plot/gnuplot.h @@ -7,13 +7,14 @@ */ #pragma once - +#include #include "interop/model/plot/flowcell_data.h" #include "interop/model/plot/bar_point.h" #include "interop/model/plot/candle_stick_point.h" #include "interop/model/plot/plot_data.h" -namespace illumina { namespace interop { namespace io { namespace plot { +namespace illumina { namespace interop { namespace io { namespace plot +{ /** Write a plot in the GNUPlot scripting language */ @@ -26,9 +27,9 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param data flowcell heatmap data * @param output_image_path optional output image path for the script */ - void write_flowcell(std::ostream& out, - const model::plot::flowcell_data& data, - const std::string& output_image_path="") + void write_flowcell(std::ostream &out, + const model::plot::flowcell_data &data, + const std::string &output_image_path = "") { write_output_format(out, output_image_path); write_title(out, data); @@ -37,18 +38,19 @@ namespace illumina { namespace interop { namespace io { namespace plot { out << "unset tics\n"; out << "unset border\n"; out << "set cbrange [" << data.saxis().min() << ":" << data.saxis().max() << "]\n"; - out << "set palette defined (0 \"blueviolet\", 0.143 \"blue\", 0.286 \"aqua\", 0.429 \"lightgreen\", 0.572 \"limegreen\", 0.715 \"lime\", 0.858 \"yellow\", 1 \"orange\")\n"; + out << + "set palette defined (0 \"blueviolet\", 0.143 \"blue\", 0.286 \"aqua\", 0.429 \"lightgreen\", 0.572 \"limegreen\", 0.715 \"lime\", 0.858 \"yellow\", 1 \"orange\")\n"; out << "set autoscale fix\n"; out << "set size ratio 2\n"; out << "plot \"-\" matrix with image" << "\n"; - const size_t swath_count = data.column_count()/data.tile_count(); - for(size_t y=0;y - void write_chart(std::ostream& out, - const model::plot::plot_data& data, - const std::string& output_image_path="") + void write_chart(std::ostream &out, + const model::plot::plot_data &data, + const std::string &output_image_path = "") { write_output_format(out, output_image_path); write_title(out, data); @@ -135,26 +139,27 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param data plot data for a line, candlestick or bar plot */ template - void write_plot_data(std::ostream& out, const model::plot::plot_data& data) + void write_plot_data(std::ostream &out, const model::plot::plot_data &data) { - if(data.size() == 0) return; - for(size_t series=0;series < data.size();++series) + if (data.size() == 0) return; + for (size_t series = 0; series < data.size(); ++series) { write_plot_series(out, data[series]); write_end_of_series(out); // Marker for the end of the series so GNUPlot knows when the next line starts } } + /** Write a description of the plot data to the output stream * * @param out output stream * @param data plot data for a line, candlestick or bar plot */ template - void write_plot_description(std::ostream& out, const model::plot::plot_data& data) + void write_plot_description(std::ostream &out, const model::plot::plot_data &data) { - if(data.size() == 0) return; + if (data.size() == 0) return; - switch(data[0].series_type()) + switch (data[0].series_type()) { case model::plot::series::Bar: out << "set style fill solid border -1\n"; @@ -168,48 +173,52 @@ namespace illumina { namespace interop { namespace io { namespace plot { }; out << "plot "; - for(size_t series=0;series < data.size();++series) + for (size_t series = 0; series < data.size(); ++series) { out << "\"-\" "; write_type(out, data[series]); write_label(out, data[series]); write_color(out, data[series]); write_additional(out, data[series]); - if((series+1) < data.size()) out << ","; + if ((series + 1) < data.size()) out << ","; } out << "\n"; } + /** Write the label and range for each axis to the output stream * * @param out output stream * @param axes pair of axis objects */ - void write_axes(std::ostream& out, const model::plot::axes& axes) + void write_axes(std::ostream &out, const model::plot::axes &axes) { write_axis(out, axes.y(), 'y'); write_axis(out, axes.x(), 'x'); } + /** Write the label and range for an axis to the output stream * * @param out output stream * @param axis range and label of the axis * @param axis_label 'x' or 'y' to denote either x- or y-axis */ - void write_axis(std::ostream& out, const model::plot::axis& axis, const char axis_label) + void write_axis(std::ostream &out, const model::plot::axis &axis, const char axis_label) { out << "set " << axis_label << "range [" << axis.min() << " : " << axis.max() << " ]\n"; - if(axis.label() != "") out << "set " << axis_label << "label \"" << axis.label() << "\"\n"; + if (axis.label() != "") out << "set " << axis_label << "label \"" << axis.label() << "\"\n"; } + /** Write the title of the plot to the output stream * * @param out output stream * @param data plot data for a line, candlestick, bar plot, heat map or flowcell heat map */ template - void write_title(std::ostream& out, const PlotData& data) + void write_title(std::ostream &out, const PlotData &data) { if (data.title() != "") out << "set title \"" << data.title() << "\"\n"; } + /** Write the output format for GNUPlot * * This tells GNUPlot to write the plot as an image in PNG format @@ -217,9 +226,9 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param out output stream * @param output_image_path output filename of the image */ - void write_output_format(std::ostream& out, const std::string& output_image_path) + void write_output_format(std::ostream &out, const std::string &output_image_path) { - if(output_image_path == "") return; + if (output_image_path == "") return; out << "set terminal png crop" << std::endl; out << "set output \'" << output_image_path << "\'" << std::endl; //out << "set style fill solid noborder" << std::endl; @@ -231,13 +240,13 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param out output stream * @param series series of candlestick points */ - void write_plot_series(std::ostream& out, const model::plot::series< model::plot::candle_stick_point >& series) + void write_plot_series(std::ostream &out, const model::plot::series &series) { - if(series.series_type() == model::plot::series::Candlestick) + if (series.series_type() == model::plot::series::Candlestick) { - for(size_t i=0;i - void write_plot_line(std::ostream& out, const model::plot::series

& series) + void write_plot_line(std::ostream &out, const model::plot::series

&series) { - for(size_t i=0;i - void write_plot_series(std::ostream& out, const model::plot::series< model::plot::data_point >& series) + void write_plot_series(std::ostream &out, const model::plot::series > &series) { write_plot_line(out, series); } + /** Write a series of bar plot points to the output stream * * @param out output stream * @param series series of bar plot points */ - void write_plot_series(std::ostream& out, const model::plot::series& series) + void write_plot_series(std::ostream &out, const model::plot::series &series) { - for(size_t i=0;i - void write_type(std::ostream& out, - const model::plot::series

& series, - const char sep=' ') + void write_type(std::ostream &out, + const model::plot::series

&series, + const char sep = ' ') { - switch(series.series_type()) + switch (series.series_type()) { case model::plot::series

::Bar: out << "using 1:2:3 with boxes" << sep; @@ -319,6 +332,7 @@ namespace illumina { namespace interop { namespace io { namespace plot { break; } } + /** Write additional features of the plot based on the plot type * * This function determines the plot type from the series_type() function @@ -328,9 +342,9 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param sep seperator between features */ template - void write_additional(std::ostream& out, const model::plot::series

& series, const char sep=' ') + void write_additional(std::ostream &out, const model::plot::series

&series, const char sep = ' ') { - switch(series.series_type()) + switch (series.series_type()) { case model::plot::series

::Candlestick: out << "whiskerbars" << sep;//, \'\' using 1:4:4:4:4 with candlesticks" << sep; @@ -339,6 +353,7 @@ namespace illumina { namespace interop { namespace io { namespace plot { break; } } + /** Write the label for the series * * If there is no label, notitle is written instead. @@ -348,12 +363,13 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param sep seperator between features */ template - void write_label(std::ostream& out, const model::plot::series

& series, const char sep=' ') + void write_label(std::ostream &out, const model::plot::series

&series, const char sep = ' ') { - if(series.title() == "") out << "notitle"; + if (series.title() == "") out << "notitle"; else out << "title \"" << series.title() << "\""; out << sep; } + /** Write the color of the series * * @@ -362,25 +378,26 @@ namespace illumina { namespace interop { namespace io { namespace plot { * @param sep separator between features */ template - void write_color(std::ostream& out, const model::plot::series

& series, const char sep=' ') + void write_color(std::ostream &out, const model::plot::series

&series, const char sep = ' ') { - if(series.color() == "") return; - out << "lt rgb \"" << normalize_color(series.color()) << "\""; + if (series.color() == "") return; + out << "lt rgb \"" << normalize_color(series.color()) << "\""; out << sep; } + /** Normalize the camel case color to lower case and seperated with a dash (`-`) * * @param color camel case color name * @return GNUPlot color name */ - std::string normalize_color(const std::string& color) + std::string normalize_color(const std::string &color) { std::string color_normalized; - for(size_t i=0;i 0) color_normalized += '-'; + if (i > 0) color_normalized += '-'; color_normalized += static_cast(::tolower(color[i])); } else color_normalized += color[i]; @@ -390,5 +407,4 @@ namespace illumina { namespace interop { namespace io { namespace plot { }; - }}}} diff --git a/interop/io/stream_exceptions.h b/interop/io/stream_exceptions.h index feb25d855..c7168e03a 100644 --- a/interop/io/stream_exceptions.h +++ b/interop/io/stream_exceptions.h @@ -3,7 +3,6 @@ * The header provides exceptions for specific errors that may be raised while reading a binary InterOp file. * * @file - * * @date 8/9/15 * @version 1.0 * @copyright GNU Public License. @@ -13,7 +12,6 @@ #include #include -#include "interop/util/exception_specification.h" #ifdef _MSC_VER #pragma warning(disable:4290) // MSVC warns that it ignores the exception specification. @@ -86,13 +84,13 @@ namespace illumina { namespace interop { namespace io * * This is only raised if the record is read from the stream incomplete */ - struct invalid_argument : public std::invalid_argument + struct invalid_argument : public std::runtime_error { /** Constructor * * @param mesg error message */ - invalid_argument(const std::string &mesg) : std::invalid_argument(mesg) + invalid_argument(const std::string &mesg) : std::runtime_error(mesg) { } }; /** @} */ diff --git a/interop/io/table/csv_format.h b/interop/io/table/csv_format.h index d6b7a3612..f36ba167b 100644 --- a/interop/io/table/csv_format.h +++ b/interop/io/table/csv_format.h @@ -9,14 +9,16 @@ #include "interop/util/lexical_cast.h" -namespace illumina { namespace interop { namespace io { namespace table { +namespace illumina { namespace interop { namespace io { namespace table +{ /** Read a vector of values from a single in a CSV file * * @param in input stream * @param values destination vector + * @param missing sentinel for missing values */ template - void read_csv_line(std::istream& in, std::vector& values) + void read_csv_line(std::istream& in, std::vector& values, const T missing=T()) { values.clear(); std::string line; @@ -25,7 +27,7 @@ namespace illumina { namespace interop { namespace io { namespace table { std::string cell; while(std::getline(sin, cell, ',')) { - if(cell=="") values.push_back(std::numeric_limits::min()); + if(cell=="") values.push_back(missing); else values.push_back(util::lexical_cast(cell)); } } @@ -33,15 +35,19 @@ namespace illumina { namespace interop { namespace io { namespace table { * * @param out output stream * @param values source vector of values + * @param beg start column offset + * @param last last column offset */ template - void write_csv_line(std::ostream& out, const std::vector& values) + void write_csv_line(std::ostream& out, const std::vector& values, const size_t beg=0, size_t last=0) { if(values.empty())return; - out << values[0]; - for(typename std::vector::const_iterator it=values.begin()+1, end=values.end();it != end;++it) + if(last == 0) last = values.size(); + INTEROP_ASSERT(beg < values.size()); + out << values[beg]; + INTEROP_ASSERT(last <= values.size()); + for(typename std::vector::const_iterator it=values.begin()+beg+1, end=values.begin()+last;it != end;++it) { - out << "," << *it; } out << "\n"; diff --git a/interop/io/table/imaging_table_csv.h b/interop/io/table/imaging_table_csv.h new file mode 100644 index 000000000..510dfaefa --- /dev/null +++ b/interop/io/table/imaging_table_csv.h @@ -0,0 +1,112 @@ +/** Write an imaging table into a CSV file + * + * @file + * @date 7/21/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/io/table/csv_format.h" +#include "interop/model/table/imaging_table.h" +#include "interop/logic/table/create_imaging_table_columns.h" +#include "interop/logic/table/create_imaging_table.h" + +namespace illumina { namespace interop { namespace model { namespace table +{ + + + /** Read a column header from an input stream + * + * @param in input stream + * @param column column header + * @return input stream + */ + inline std::istream& operator>>(std::istream& in, imaging_column& column) + { + std::getline(in, column.m_name, ','); + std::string::size_type n; + if((n=column.m_name.find("<")) != std::string::npos) + { + column.m_subcolumn_names.clear(); + std::istringstream iss(column.m_name.substr(n+1, column.m_name.length()-1-n-1)); + std::string tmp; + while(iss.good()) + { + std::getline(iss, tmp, ';'); + if(!tmp.empty()) column.m_subcolumn_names.push_back(tmp); + else break; + } + column.m_name = column.m_name.substr(0, n); + } + return in; + } + + /** Write a column header to an output stream + * + * @param out output stream + * @param column column header + * @return output stream + */ + inline std::ostream& operator<<(std::ostream& out, const imaging_column& column) + { + if(column.has_children()) + { + out << column.name() << "<" << column.subcolumns()[0]; + for(size_t i=1;i"; + } + else out << column.name(); + return out; + } + + /** Read an imaging table from an input stream in the CSV format + * + * @param in input stream + * @param table imaging table + * @return input stream + */ + inline std::istream &operator>>(std::istream &in, imaging_table &table) + { + imaging_table::column_vector_t cols; + io::table::read_csv_line(in, cols); + if (!in.good()) return in; + logic::table::populate_column_offsets(cols); + + const size_t column_count = logic::table::count_table_columns(cols); + size_t row_count = 0; + imaging_table::data_vector_t data; + imaging_table::data_vector_t values; + values.reserve(column_count); + while (!in.eof()) + { + io::table::read_csv_line(in, values, std::numeric_limits::quiet_NaN()); + if (values.empty()) continue; + if(column_count != values.size()) + INTEROP_THROW(io::bad_format_exception, "Number of values does not match number of columns"); + data.resize(data.size()+column_count); + std::copy(values.begin(), values.end(), data.begin()+data.size()-column_count); + ++row_count; + } + table.set_data(row_count, cols, data); + return in; + } + /** Write the imaging table to the output stream in the CSV format + * + * @param out output stream + * @param table imaging table + * @return output stream + */ + inline std::ostream &operator<<(std::ostream &out, const imaging_table &table) + { + if (!out.good()) return out; + io::table::write_csv_line(out, table.m_columns); + if (!out.good()) return out; + for (size_t row=0,offset=0;row& metrics, + void copy_focus(const model::metric_base::metric_set& metrics, float *focus_scores, const size_t channel, - const size_t n) throw(model::invalid_parameter) - { - typedef model::metric_base::metric_set::const_iterator const_iterator; - if(metrics.size()==0)return; - if(n < metrics.size()) throw model::invalid_parameter("Buffer size too small for metric set"); - if(channel >= metrics.metrics()[0].channel_count()) - throw model::invalid_parameter("Channel exceeds channel count"); - for (const_iterator it = metrics.begin(); it != metrics.end(); ++it, ++focus_scores) - *focus_scores = it->focus_score(channel); - } + const size_t n) throw(model::invalid_parameter, model::index_out_of_bounds_exception); }}}} diff --git a/interop/logic/metric/metric_value.h b/interop/logic/metric/metric_value.h index 97f1665f1..c49ebc284 100644 --- a/interop/logic/metric/metric_value.h +++ b/interop/logic/metric/metric_value.h @@ -18,7 +18,8 @@ #include "interop/model/metrics/corrected_intensity_metric.h" -namespace illumina { namespace interop { namespace logic { namespace metric { +namespace illumina { namespace interop { namespace logic { namespace metric +{ /** This template class retrieves a value from a metric object based on the value of the * provided metric_type enum and possibly an index or other information. @@ -211,6 +212,8 @@ namespace illumina { namespace interop { namespace logic { namespace metric { return metric.corrected_int_called(base); case constants::SignalToNoise: return metric.signal_to_noise(); + case constants::PercentNoCall: + return metric.percent_nocall(); default: INTEROP_THROW(model::invalid_metric_type, "Unknown metric type " << constants::to_string(type)); } diff --git a/interop/logic/metric/q_metric.h b/interop/logic/metric/q_metric.h index 1f4a5e5be..71fcf9afc 100644 --- a/interop/logic/metric/q_metric.h +++ b/interop/logic/metric/q_metric.h @@ -7,16 +7,15 @@ */ #pragma once #include -#include #include "interop/model/metrics/q_metric.h" #include "interop/model/metrics/q_collapsed_metric.h" #include "interop/model/metrics/q_by_lane_metric.h" #include "interop/model/model_exceptions.h" -#include "interop/constants/enums.h" #include "interop/model/metric_base/metric_set.h" -namespace illumina { namespace interop { namespace logic { namespace metric { +namespace illumina { namespace interop { namespace logic { namespace metric +{ /** Count the number of bins in the q metric * @@ -31,7 +30,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { } /** Count the number of bins in the q metric * - * @param q_metric_set q-metric set + * @param metric_set q-metric set * @return number of bins */ inline size_t count_q_metric_bins(const model::metric_base::metric_set& metric_set) @@ -40,7 +39,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { } /** Count the number of bins in the q metric * - * @param q_metric_set q-metric set + * @param metric_set q-metric set * @return number of bins */ inline size_t count_q_metric_bins(const model::metric_base::metric_set& metric_set) @@ -50,134 +49,41 @@ namespace illumina { namespace interop { namespace logic { namespace metric { /** Count the number of bins in the q metric * * @note Always returns 0 - * @param q_metric_set q-metric set * @return 0 */ inline size_t count_q_metric_bins(const model::metric_base::metric_set&) { return 0; } - /** Populate cumulative q-metric distribution - * - * @param metric_set q-metric set - */ - template - void populate_cumulative_distribution_t(model::metric_base::metric_set& metric_set) - throw( model::index_out_of_bounds_exception ) - { - if(metric_set.size()==0) return; - typedef QMetric q_metric; - typedef model::metric_base::metric_set q_metric_set; - typedef typename q_metric_set::id_vector id_vector; - typedef typename q_metric_set::uint_t uint_t; - typedef typename id_vector::const_iterator const_id_iterator; - id_vector lane_ids = metric_set.lanes(); - for(const_id_iterator lane_beg = lane_ids.begin(), lane_end = lane_ids.end();lane_beg != lane_end;++lane_beg) - { - const uint_t lane_id = *lane_beg; - id_vector tile_ids = metric_set.tile_numbers_for_lane(lane_id); - for(const_id_iterator tile_beg = tile_ids.begin(), tile_end = tile_ids.end();tile_beg != tile_end;++tile_beg) - { - const uint_t tile_id = *tile_beg; - size_t prev_idx = metric_set.find(lane_id, tile_id, 1); - if(prev_idx >= metric_set.size()) continue; - QMetric& metric = metric_set.at(prev_idx); - metric.accumulate(metric); - const uint_t second_cycle_start = 2; // We have to accumulate the first cycle with itself, and every - // subsequent with the previous cycle. - // Also this is not 0-indexed, so we start with 2, the 2nd cycle - for(uint_t cycle = second_cycle_start;cycle <= metric_set.max_cycle();++cycle) - { - const size_t cur_idx = metric_set.find(lane_id, tile_id, cycle); - if(cur_idx>=metric_set.size() || prev_idx>=metric_set.size()) - continue;// TODO: if this happens zero out following q-scores - metric_set.at(cur_idx).accumulate(metric_set.at(prev_idx)); - prev_idx=cur_idx; - } - } - } - } /** Populate cumulative by lane q-metric distribution * * @param q_metric_set q-metric set */ - inline void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) - throw( model::index_out_of_bounds_exception ) - { - if(q_metric_set.size()==0) return; - typedef model::metrics::q_by_lane_metric q_by_lane_metric; - typedef model::metric_base::metric_set q_by_lane_metric_set; - typedef q_by_lane_metric_set::id_vector id_vector; - typedef q_by_lane_metric_set::uint_t uint_t; - id_vector lane_ids = q_metric_set.lanes(); - const size_t tile_id = 0; - for(id_vector::const_iterator lane_beg = lane_ids.begin(), lane_end = lane_ids.end();lane_beg != lane_end;++lane_beg) - { - size_t prev_idx = q_metric_set.find(*lane_beg, tile_id, 1); - if(prev_idx >= q_metric_set.size()) continue; - q_by_lane_metric& metric = q_metric_set.at(prev_idx); - metric.accumulate(metric); - const uint_t second_cycle_start = 2; // We have to accumulate the first cycle with itself, and every - // subsequent with the previous cycle. - // Also this is not 0-indexed, so we start with 2, the 2nd cycle - for(uint_t cycle = second_cycle_start;cycle <= q_metric_set.max_cycle();++cycle) - { - const size_t cur_idx = q_metric_set.find(*lane_beg, tile_id, cycle); - if(cur_idx>=q_metric_set.size() || prev_idx>=q_metric_set.size()) - continue;// TODO: if this happens zero out following q-scores - q_metric_set.at(cur_idx).accumulate(q_metric_set.at(prev_idx)); - prev_idx=cur_idx; - } - } - } + void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) + throw( model::index_out_of_bounds_exception ); /** Populate cumulative q-metric distribution * * @note This can exist here or in SWIG. This is a swig interface function. * @param q_metric_set q-metric set */ - inline void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) - throw( model::index_out_of_bounds_exception ) - { - populate_cumulative_distribution_t(q_metric_set); - } + void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) + throw( model::index_out_of_bounds_exception ); /** Populate cumulative cpllapsed q-metric distribution * * @note This can exist here or in SWIG. This is a swig interface function. * @param q_metric_set q-metric set */ - inline void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) - throw( model::index_out_of_bounds_exception ) - { - populate_cumulative_distribution_t(q_metric_set); - } + void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) + throw( model::index_out_of_bounds_exception ); /** Count number of unique counts to determine number * of unique bins for legacy binning * - * @Note, if the number of bins is greater than 7, than this function stops counting! + * @note, if the number of bins is greater than 7, than this function stops counting! * * @param q_metric_set q-metric set * @return number of unique bins */ - inline size_t count_legacy_q_score_bins(const model::metric_base::metric_set& q_metric_set) - { - // 0 is a sentinel that indicates legacy binning is not required - if(q_metric_set.version() > 4) return 0; // Version 5 and later do not require legacy binning - if(!q_metric_set.bins().empty()) return 0; // If the metrics already have a header they do not require binning - - const size_t max_bin_count = 7; - model::metric_base::metric_set::const_iterator beg = q_metric_set.begin(), - end=q_metric_set.end(); - if(beg == end) return 0 ; - typedef model::metrics::q_metric::uint_t uint_t; - std::set bins_found; - for(;beg != end;++beg) - { - for(uint_t i=0;i(beg->qscoreHist().size());++i) - if(beg->qscoreHist()[i] > 0) bins_found.insert(i); - if(bins_found.size() > max_bin_count) break; // Number of bins greater than 7 indicates this is unbinned - } - return bins_found.size(); - } + size_t count_legacy_q_score_bins(const model::metric_base::metric_set& q_metric_set); /** Test if legacy binning should be performed * * @param count number of bins @@ -196,72 +102,9 @@ namespace illumina { namespace interop { namespace logic { namespace metric { * @param instrument instrument type * @param count number of bins */ - inline void populate_legacy_q_score_bins(std::vector& q_score_bins, + void populate_legacy_q_score_bins(std::vector& q_score_bins, const constants::instrument_type instrument, - const size_t count) - { - typedef model::metrics::q_score_bin q_score_bin; - if(!requires_legacy_bins(count)) return; - q_score_bins.reserve(count); - if(instrument == constants::NextSeq) - { - q_score_bins.push_back(q_score_bin(0, 9, 8)); - q_score_bins.push_back(q_score_bin(10, 19, 13)); - q_score_bins.push_back(q_score_bin(20, 24, 22)); - q_score_bins.push_back(q_score_bin(25, 29, 27)); - q_score_bins.push_back(q_score_bin(30, 34, 32)); - q_score_bins.push_back(q_score_bin(35, 39, 37)); - } - else if(count == 7) - { - q_score_bins.push_back(q_score_bin(0, 9, 6)); - q_score_bins.push_back(q_score_bin(10, 19, 15)); - q_score_bins.push_back(q_score_bin(20, 24, 22)); - q_score_bins.push_back(q_score_bin(25, 29, 27)); - q_score_bins.push_back(q_score_bin(30, 34, 33)); - q_score_bins.push_back(q_score_bin(35, 39, 37)); - q_score_bins.push_back(q_score_bin(40, 49, 40)); - } - else if(count == 6) - { - q_score_bins.push_back(q_score_bin(0, 9, 7)); - q_score_bins.push_back(q_score_bin(10, 19, 16)); - q_score_bins.push_back(q_score_bin(20, 26, 24)); - q_score_bins.push_back(q_score_bin(27, 29, 29)); - q_score_bins.push_back(q_score_bin(30, 34, 33)); - q_score_bins.push_back(q_score_bin(35, 49, 38)); - } - else if(count == 5) - { - q_score_bins.push_back(q_score_bin(0, 9, 7)); - q_score_bins.push_back(q_score_bin(10, 19, 16)); - q_score_bins.push_back(q_score_bin(20, 29, 25)); - q_score_bins.push_back(q_score_bin(30, 34, 33)); - q_score_bins.push_back(q_score_bin(35, 49, 38)); - } - else if(count == 4) - { - q_score_bins.push_back(q_score_bin(0, 9, 7)); - q_score_bins.push_back(q_score_bin(10, 29, 20)); - q_score_bins.push_back(q_score_bin(30, 34, 33)); - q_score_bins.push_back(q_score_bin(35, 49, 38)); - } - else if(count == 3) - { - q_score_bins.push_back(q_score_bin(0, 9, 7)); - q_score_bins.push_back(q_score_bin(10, 29, 20)); - q_score_bins.push_back(q_score_bin(30, 49, 36)); - } - else if(count == 2) - { - q_score_bins.push_back(q_score_bin(0, 27, 13)); - q_score_bins.push_back(q_score_bin(28, 49, 35)); - } - else - { - q_score_bins.push_back(q_score_bin(0, 50, 20)); - } - } + const size_t count); /** Populate the q-score header bins from the data * * This only for legacy platforms that use older q-metric formats, which do not include bin information @@ -427,53 +270,15 @@ namespace illumina { namespace interop { namespace logic { namespace metric { * @param metric_set q-metric set * @param collapsed collapsed Q-metrics */ - inline void create_collapse_q_metrics(const model::metric_base::metric_set& metric_set, - model::metric_base::metric_set& collapsed) - { - typedef model::metric_base::metric_set::const_iterator const_iterator; - typedef model::metric_base::metric_set::uint_t uint_t; - - const uint_t q20_idx = static_cast(index_for_q_value(metric_set, 20)); - const uint_t q30_idx = static_cast(index_for_q_value(metric_set, 30)); - - collapsed.set_version(metric_set.version()); - for(const_iterator beg = metric_set.begin(), end = metric_set.end();beg != end;++beg) - { - const uint_t q20 = beg->total_over_qscore(q20_idx); - const uint_t q30 = beg->total_over_qscore(q30_idx); - const uint_t total = beg->sum_qscore(); - const uint_t median = beg->median(metric_set.bins()); - collapsed.insert(model::metrics::q_collapsed_metric(beg->lane(), - beg->tile(), - beg->cycle(), - q20, - q30, - total, - median)); - } - } + void create_collapse_q_metrics(const model::metric_base::metric_set& metric_set, + model::metric_base::metric_set& collapsed); /** Generate by lane Q-metric data from Q-metrics * * @param metric_set Q-metrics - * @param collapsed collapsed Q-metrics + * @param bylane bylane Q-metrics + * @throws index_out_of_bounds_exception */ - inline void create_q_metrics_by_lane(const model::metric_base::metric_set& metric_set, + void create_q_metrics_by_lane(const model::metric_base::metric_set& metric_set, model::metric_base::metric_set& bylane) - throw(model::index_out_of_bounds_exception) - { - typedef model::metric_base::metric_set::const_iterator const_iterator; - typedef model::metric_base::metric_set::header_type header_type; - typedef model::metric_base::base_cycle_metric::id_t id_t; - - bylane = static_cast(metric_set); - bylane.set_version(metric_set.version()); - for(const_iterator beg = metric_set.begin(), end = metric_set.end();beg != end;++beg) - { - const id_t id = model::metric_base::base_cycle_metric::id(beg->lane(), 0, beg->cycle()); - if(bylane.has_metric(id)) - bylane.get_metric_ref(id).accumulate_by_lane(*beg); - else - bylane.insert(model::metrics::q_by_lane_metric(beg->lane(), 0, beg->cycle(), beg->qscore_hist())); - } - } + throw(model::index_out_of_bounds_exception); }}}} diff --git a/interop/logic/metric/tile_metric.h b/interop/logic/metric/tile_metric.h index 59b66785e..78ab76ddf 100644 --- a/interop/logic/metric/tile_metric.h +++ b/interop/logic/metric/tile_metric.h @@ -12,7 +12,33 @@ #include "interop/model/run/flowcell_layout.h" -namespace illumina { namespace interop { namespace logic { namespace metric { +namespace illumina { namespace interop { namespace logic { namespace metric +{ + + /** Convert string lane/tile identifier (1_1111) to lane number + * + * @param tile_name lane/tile identifier (1_1111) + * @return lane number (1) + */ + inline ::uint32_t lane_from_name(const std::string& tile_name) + { + if(tile_name=="") return 0; + const size_t n = tile_name.find('_'); + if(n == std::string::npos) return 0; + return util::lexical_cast< ::uint32_t >(tile_name.substr(0, n)); + } + /** Convert string lane/tile identifier (1_1111) to tile hash + * + * @param tile_name lane/tile identifier (1_1111) + * @return tile hash (1111) + */ + inline ::uint32_t tile_from_name(const std::string& tile_name) + { + if(tile_name=="") return 0; + const size_t n = tile_name.find('_'); + if(n == std::string::npos) return 0; + return util::lexical_cast< ::uint32_t >(tile_name.substr(n+1)); + } /** Determine the tile naming method from a metric tile ID * @@ -27,7 +53,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { } /** Determine the tile naming method from a metric set * - * @param metric tile metric set + * @param metric_set metric set * @return tile_naming_method */ template diff --git a/interop/logic/plot/plot_by_cycle.h b/interop/logic/plot/plot_by_cycle.h index 8fb644ed2..66a2a1366 100644 --- a/interop/logic/plot/plot_by_cycle.h +++ b/interop/logic/plot/plot_by_cycle.h @@ -6,143 +6,14 @@ * @copyright GNU Public License. */ #pragma once -#include "interop/util/exception.h" -#include "interop/util/statistics.h" -#include "interop/constants/enums.h" -#include "interop/model/metrics/extraction_metric.h" -#include "interop/model/metrics/corrected_intensity_metric.h" -#include "interop/model/model_exceptions.h" +#include "interop/model/run_metrics.h" #include "interop/model/plot/filter_options.h" -#include "interop/model/plot/series.h" #include "interop/model/plot/plot_data.h" -#include "interop/model/run_metrics.h" -#include "interop/logic/utils/metric_type_ext.h" -#include "interop/logic/utils/channel.h" -#include "interop/logic/metric/metric_value.h" -#include "interop/logic/plot/plot_point.h" -#include "interop/logic/plot/plot_data.h" +#include "interop/model/plot/candle_stick_point.h" +#include "interop/constants/enums.h" namespace illumina { namespace interop { namespace logic { namespace plot { - - /** Plot the average over all tiles of a specific metric by cycle - * - * @param metrics set of metric records - * @param proxy functor that takes a metric record and returns a metric value - * @param options filter for metric records - * @param type type of metric to extract using the proxy functor - * @param points collection of points where x is cycle number and y is the mean metric value - */ - template - size_t populate_metric_average_by_cycle(const MetricSet& metrics, - MetricProxy& proxy, - const model::plot::filter_options& options, - const constants::metric_type type, - model::plot::data_point_collection& points) - { - const size_t max_cycle = metrics.max_cycle(); - points.assign(max_cycle, Point()); - const float dummy_x = 1; - for(typename MetricSet::const_iterator b = metrics.begin(), e = metrics.end();b != e;++b) - { - if(!options.valid_tile(*b)) continue; - const float val = proxy(*b, type); - if(std::isnan(val)) continue; - points[b->cycle()-1].add(dummy_x, val); - } - size_t index = 0; - for(size_t cycle=0;cycle(points[cycle].x()) == 0) continue; - const float avg = points[cycle].y()/points[cycle].x(); - points[index].set(static_cast(cycle+1), avg); - ++index; - } - points.resize(index); - return max_cycle; - } - /** Plot the candle stick over all tiles of a specific metric by cycle - * - * @param metrics set of metric records - * @param proxy functor that takes a metric record and returns a metric value - * @param options filter for metric records - * @param type type of metric to extract using the proxy functor - * @param points collection of points where x is cycle number and y is the candle stick metric values - * @return last populated cycle - */ - template - size_t populate_candle_stick_by_cycle(const MetricSet& metrics, - MetricProxy& proxy, - const model::plot::filter_options& options, - const constants::metric_type type, - model::plot::data_point_collection& points) - { - const size_t max_cycle = metrics.max_cycle(); - const size_t tile_count = static_cast(std::ceil(static_cast(metrics.size())/max_cycle)); - std::vector< std::vector > tile_by_cycle(max_cycle); - for(size_t i=0;i outliers; - outliers.reserve(10); // TODO: use as flag for keeping outliers - - for(typename MetricSet::const_iterator b = metrics.begin(), e = metrics.end();b != e;++b) - { - if(!options.valid_tile(*b)) continue; - const float val = proxy(*b, type); - if(std::isnan(val)) continue; - tile_by_cycle[b->cycle()-1].push_back(val); - } - points.resize(max_cycle); - size_t j=0; - for(size_t cycle=0;cycle(cycle+1), - outliers); - ++j; - } - points.resize(j); - return max_cycle; - } - /** Generate meta data for multiple plot series that compare data by channel - * - * @param channels collection of channel names - * @param data plot data that holds a collection of plot series - */ - template - void setup_series_by_channel(const std::vector& channels, model::plot::plot_data& data) - { - typedef model::plot::series series_t; - data.resize(channels.size()); - std::vector expected_order(channels.size()); - utils::actual2expected(channels, expected_order); - for(size_t i=0;i(expected_order[i])), - series_t::Line); - } - } - /** Generate meta data for multiple plot series that compare data by base - * - * @param data plot data that holds a collection of plot series - */ - template - void setup_series_by_base(model::plot::plot_data& data) - { - typedef model::plot::series series_t; - data.resize(constants::NUM_OF_BASES); - for(size_t i=0;i(i)), - constants::to_string(static_cast(i)), - series_t::Line); - } - } /** Plot a specified metric value by cycle * * @ingroup plot_logic @@ -151,153 +22,19 @@ namespace illumina { namespace interop { namespace logic { namespace plot * @param options options to filter the data * @param data output plot data */ - template void plot_by_cycle(model::metrics::run_metrics& metrics, const constants::metric_type type, const model::plot::filter_options& options, - model::plot::plot_data& data) + model::plot::plot_data& data) throw(model::index_out_of_bounds_exception, model::invalid_metric_type, model::invalid_channel_exception, model::invalid_filter_option, - model::invalid_read_exception) - { - data.clear(); - if(!options.all_cycles()) - INTEROP_THROW(model::invalid_filter_option, "Filtering by cycle is not supported"); - if(!options.all_reads()) - INTEROP_THROW(model::invalid_filter_option, "Filtering by read is not supported"); - if(!utils::is_cycle_metric(type)) - INTEROP_THROW(model::invalid_filter_option, "Only cycle metrics are supported"); - options.validate(type, metrics.run_info()); - size_t max_cycle=0; - switch(logic::utils::to_group(type)) - { - case constants::Extraction: - { - if(options.all_channels(type)) - { - setup_series_by_channel(metrics.run_info().channels(), data); - for(size_t i=0;i proxy(i); - max_cycle = populate_metric_average_by_cycle( - metrics.get_set(), - proxy, - options, - type, - data[i] - ); - } - } - else - { - data.assign(1, model::plot::series()); - const size_t channel = options.channel(); - metric::metric_value proxy(channel); - max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), - proxy, - options, - type, - data[0]); - } - break; - } - case constants::CorrectedInt: - { - if(options.all_bases(type)) - { - setup_series_by_base(data); - for(size_t i=0;i proxy( - static_cast(i)); - max_cycle = populate_metric_average_by_cycle( - metrics.get_set(), - proxy, - options, - type, - data[i] - ); - } - } - else - { - data.assign(1, model::plot::series()); - const constants::dna_bases base = options.dna_base(); - metric::metric_value proxy(base); - max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), - proxy, - options, - type, - data[0]); - } - break; - } - case constants::Q: - { - data.assign(1, model::plot::series()); + model::invalid_read_exception); - typedef model::metrics::q_collapsed_metric metric_t; - metric::metric_value proxy2; - if(0 == metrics.get_set().size()) - logic::metric::create_collapse_q_metrics(metrics.get_set(), - metrics.get_set()); - max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), - proxy2, - options, - type, - data[0]); - break; - } - case constants::Error://TODO: skip last cycle of read for error metric - { - data.assign(1, model::plot::series()); - metric::metric_value proxy3; - max_cycle = populate_candle_stick_by_cycle( - metrics.get_set(), - proxy3, - options, - type, - data[0]); - break; - } - default: - INTEROP_THROW(model::invalid_metric_type, "Invalid metric group"); - } - if(type != constants::FWHM) - { - auto_scale_y(data); - if(type == constants::ErrorRate) - { - data.set_yrange(0, std::min(5.0f, data.y_axis().max())); - } - else if(type == constants::BasePercent) - { - data.set_yrange(0, std::max(50.0f, data.y_axis().max())); - } - } - else data.set_yrange(0, 6); - data.set_xrange(0, static_cast(max_cycle+2)); - data.set_xlabel("Cycle"); - data.set_ylabel(utils::to_description(type)); - std::string title = metrics.run_info().flowcell().barcode(); - if(title != "") title += " "; - title += options.lane_description(); - if(logic::utils::is_channel_metric(type)) - title += " " + options.channel_description(metrics.run_info().channels()); - if(logic::utils::is_base_metric(type)) - title += " " + options.base_description(); - if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) - title += " " + options.surface_description(); - data.set_title(title); - } - /** Plot a specified metric value by cycle + /** Plot a specified metric value by cycle using the candle stick model * * @ingroup plot_logic * @todo Is this temporary? @@ -306,36 +43,25 @@ namespace illumina { namespace interop { namespace logic { namespace plot * @param options options to filter the data * @param data output plot data */ - template void plot_by_cycle(model::metrics::run_metrics& metrics, const std::string& metric_name, const model::plot::filter_options& options, - model::plot::plot_data& data) - throw(model::index_out_of_bounds_exception, - model::invalid_filter_option, - model::invalid_channel_exception, - model::invalid_metric_type) - { - const constants::metric_type type = constants::parse(metric_name); - if(type == constants::UnknownMetricType) - INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); - plot_by_cycle(metrics, type, options, data); - } + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_filter_option, + model::invalid_channel_exception, + model::invalid_metric_type); + + /** List metric types available for by cycle plots + * + * @param types destination vector to fill with metric types + * @param ignore_accumulated if true, ignore accumulated Q20 and Q30 + */ + void list_by_cycle_metrics(std::vector& types, const bool ignore_accumulated=false); /** List metric type names available for by cycle plots * * @param names destination vector to fill with metric type names + * @param ignore_accumulated if true, ignore accumulated Q20 and Q30 */ - inline void list_by_cycle_metrics(std::vector& names) - { - std::vector types; - constants::list_enums(types); - names.clear(); - names.reserve(types.size()); - for(size_t i=0;i& names, const bool ignore_accumulated=false); +}}}} diff --git a/interop/logic/plot/plot_by_lane.h b/interop/logic/plot/plot_by_lane.h index 05c2ab575..1536191b2 100644 --- a/interop/logic/plot/plot_by_lane.h +++ b/interop/logic/plot/plot_by_lane.h @@ -7,61 +7,14 @@ */ #pragma once -#include "interop/util/exception.h" -#include "interop/util/statistics.h" #include "interop/constants/enums.h" -#include "interop/model/model_exceptions.h" #include "interop/model/run_metrics.h" #include "interop/model/plot/filter_options.h" -#include "interop/model/metrics/tile_metric.h" -#include "interop/logic/metric/metric_value.h" -#include "interop/logic/plot/plot_point.h" -#include "interop/logic/plot/plot_data.h" +#include "interop/model/plot/plot_data.h" +#include "interop/model/plot/candle_stick_point.h" -namespace illumina { namespace interop { namespace logic { namespace plot { - - - /** Plot the candle stick over all tiles of a specific metric by lane - * - * @param metrics set of metric records - * @param proxy functor that takes a metric record and returns a metric value - * @param options filter for metric records - * @param type type of metric to extract using the proxy functor - * @param points collection of points where x is lane number and y is the candle stick metric values - */ - template - void populate_candle_stick_by_lane(const MetricSet& metrics, - MetricProxy& proxy, - const model::plot::filter_options& options, - const constants::metric_type type, - model::plot::data_point_collection& points) - { - if(metrics.max_lane() == 0) return; - const size_t lane_count = metrics.max_lane(); - const size_t tile_count = static_cast(std::ceil(static_cast(metrics.size())/lane_count)); - std::vector< std::vector > tile_by_lane(metrics.max_lane()); - for(size_t i=0;i outliers; - outliers.reserve(10); - - for(typename MetricSet::const_iterator b = metrics.begin(), e = metrics.end();b != e;++b) - { - if(!options.valid_tile(*b)) continue; - const float val = proxy(*b, type); - if(std::isnan(val)) continue; - tile_by_lane[b->lane()-1].push_back(val); - } - points.resize(lane_count); - size_t offset=0; - for(size_t i=0;i(i+1); - plot_candle_stick(points[offset], tile_by_lane[i].begin(), tile_by_lane[i].end(), lane, outliers); - ++offset; - } - points.resize(offset); - } +namespace illumina { namespace interop { namespace logic { namespace plot +{ /** Plot a specified metric value by lane * @@ -71,58 +24,13 @@ namespace illumina { namespace interop { namespace logic { namespace plot { * @param options options to filter the data * @param data output plot data */ - template void plot_by_lane(const model::metrics::run_metrics& metrics, const constants::metric_type type, const model::plot::filter_options& options, - model::plot::plot_data& data) + model::plot::plot_data& data) throw(model::index_out_of_bounds_exception, model::invalid_metric_type, - model::invalid_filter_option) - { - if(utils::is_cycle_metric(type)) - INTEROP_THROW(model::invalid_metric_type, "Cycle metrics are unsupported"); - options.validate(type, metrics.run_info()); - data.assign(1, model::plot::series(utils::to_description(type), "Blue")); - metric::metric_value proxy3(options.read()); - populate_candle_stick_by_lane(metrics.get_set(), proxy3, options, type, - data[0]); - - const size_t read_count = metrics.run_info().reads().size(); - if(utils::is_read_metric(type) && options.all_reads() && read_count > 1) - INTEROP_THROW(model::invalid_filter_option, "All reads is unsupported for run with " << read_count); - - if(type == constants::ClusterCount || type == constants::Clusters )//constants::Density ) - { - data.push_back(model::plot::series("PF", "DarkGreen")); - const constants::metric_type second_type = - (type==constants::Clusters ? constants::ClustersPF : constants::ClusterCountPF); - //(type==constants::Density ? constants::DensityPF : constants::ClusterCountPF); - populate_candle_stick_by_lane(metrics.get_set(), proxy3, options, second_type, - data[1]); - } - - auto_scale(data, true, 1.2f); - if(type == constants::PercentPrephasing || type == constants::PercentPhasing) - data.set_yrange(0, 1); - data.set_xrange(0, data.x_axis().max()+1); - - data.set_xlabel("Lane"); - data.set_ylabel(utils::to_description(type)); - - std::string title = metrics.run_info().flowcell().barcode(); - if(options.is_specific_read(type)) - { - if(title != "") title += " "; - title += options.read_description(); - } - if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) - { - if(title != "") title += " "; - title += options.surface_description(); - } - data.set_title(title); - } + model::invalid_filter_option); /** Plot a specified metric value by cycle * @@ -133,36 +41,25 @@ namespace illumina { namespace interop { namespace logic { namespace plot { * @param options options to filter the data * @param data output plot data */ - template void plot_by_lane(const model::metrics::run_metrics& metrics, - const std::string& metric_name, - const model::plot::filter_options& options, - model::plot::plot_data& data) - throw(model::index_out_of_bounds_exception, - model::invalid_metric_type, - model::invalid_filter_option) - { - const constants::metric_type type = constants::parse(metric_name); - if(type == constants::UnknownMetricType) - INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); - plot_by_lane(metrics, type, options, data); - } + const std::string& metric_name, + const model::plot::filter_options& options, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_metric_type, + model::invalid_filter_option); + /** List metric types available for by lane plots + * + * @param types destination vector to fill with metric types + * @param ignore_pf if true, ignore density PF and cluster PF + */ + void list_by_lane_metrics(std::vector& types, const bool ignore_pf=false); /** List metric type names available for by lane plots * * @param names destination vector to fill with metric type names + * @param ignore_pf if true, ignore density PF and cluster PF */ - inline void list_by_lane_metrics(std::vector& names) - { - std::vector types; - constants::list_enums(types); - names.clear(); - names.reserve(types.size()); - for(size_t i=0;i& names, const bool ignore_pf=false); }}}} diff --git a/interop/logic/plot/plot_flowcell_map.h b/interop/logic/plot/plot_flowcell_map.h index 790d6392e..395955780 100644 --- a/interop/logic/plot/plot_flowcell_map.h +++ b/interop/logic/plot/plot_flowcell_map.h @@ -7,59 +7,15 @@ */ #pragma once -#include "interop/util/exception.h" -#include "interop/util/statistics.h" #include "interop/constants/enums.h" -#include "interop/model/model_exceptions.h" #include "interop/model/run_metrics.h" #include "interop/model/plot/filter_options.h" #include "interop/model/plot/flowcell_data.h" -#include "interop/logic/utils/metric_type_ext.h" -#include "interop/logic/utils/channel.h" -#include "interop/logic/metric/metric_value.h" -namespace illumina { namespace interop { namespace logic { namespace plot { +namespace illumina { namespace interop { namespace logic { namespace plot +{ - /** Populate the flowcell map based on the filter options - * - * @param beg iterator to start of q-metric collection - * @param end iterator to end of q-metric collection - * @param proxy functor that takes a metric record and returns a metric value - * @param layout layout of the flowcell - * @param options filter for metric records - * @param data flowcell map - */ - template - void populate_flowcell_map(I beg, - I end, - MetricProxy& proxy, - const constants::metric_type type, - const model::run::flowcell_layout& layout, - const model::plot::filter_options &options, - model::plot::flowcell_data& data, - std::vector& values_for_scaling) - { - if(beg == end) return; - const bool all_surfaces = !options.is_specific_surface(); - for (;beg != end;++beg) - { - if( !options.valid_tile_cycle(*beg) ) continue; - const float val = proxy(*beg, type); - if(std::isnan(val)) continue; - data.set_data(beg->lane()-1, - beg->physical_location_index( - layout.naming_method(), - layout.sections_per_lane(), - layout.tile_count(), - layout.swath_count(), - all_surfaces), - beg->tile(), - val); - values_for_scaling.push_back(val); - } - } - /** Plot a flowcell map * * @ingroup plot_logic @@ -67,132 +23,19 @@ namespace illumina { namespace interop { namespace logic { namespace plot { * @param type specific metric value to plot by cycle * @param options options to filter the data * @param data output flowcell map + * @param buffer preallocated memory for data + * @param tile_buffer preallocated memory for tile ids */ - inline void plot_flowcell_map(model::metrics::run_metrics& metrics, + void plot_flowcell_map(model::metrics::run_metrics& metrics, const constants::metric_type type, const model::plot::filter_options& options, - model::plot::flowcell_data& data) + model::plot::flowcell_data& data, + float* buffer=0, + ::uint32_t* tile_buffer=0) throw(model::invalid_filter_option, model::invalid_metric_type, - model::index_out_of_bounds_exception) - { - const model::run::flowcell_layout& layout = metrics.run_info().flowcell(); - data.clear(); - data.resize(layout.lane_count(), - layout.total_swaths(layout.surface_count() > 1 && !options.is_specific_surface()), - layout.tiles_per_lane()); - std::vector values_for_scaling; - values_for_scaling.reserve(data.length()); - - options.validate(type, metrics.run_info()); - - if(utils::is_cycle_metric(type) && options.all_cycles()) - INTEROP_THROW( model::invalid_filter_option, "All cycles is unsupported"); - if(utils::is_read_metric(type) && options.all_reads() && metrics.run_info().reads().size() > 1) - INTEROP_THROW( model::invalid_filter_option, "All reads is unsupported"); - switch(logic::utils::to_group(type)) - { - case constants::Tile: - { - typedef model::metrics::tile_metric metric_t; - typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); - metric::metric_value proxy(options.read()); - populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, - values_for_scaling); - break; - } - case constants::Extraction: - { - typedef model::metrics::extraction_metric metric_t; - typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); - const size_t channel = options.channel(); - if(options.all_channels(type)) - INTEROP_THROW(model::invalid_filter_option, "All channels is unsupported"); - metric::metric_value proxy(channel); - populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, - values_for_scaling); - break; - } - case constants::CorrectedInt: - { - typedef model::metrics::corrected_intensity_metric metric_t; - typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); - const constants::dna_bases base = options.dna_base(); - if(options.all_bases(type)) - INTEROP_THROW( model::invalid_filter_option, "All bases is unsupported"); - metric::metric_value proxy(base); - populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, - values_for_scaling); - break; - } - case constants::Q: - { - typedef model::metrics::q_collapsed_metric metric_t; - typedef model::metric_base::metric_set metric_set_t; - metric_set_t &metric_set = metrics.get_set(); - if(0 == metric_set.size()) - { - logic::metric::create_collapse_q_metrics(metrics.get_set(), metric_set); - if(type == constants::AccumPercentQ20 || type == constants::AccumPercentQ30) - logic::metric::populate_cumulative_distribution(metric_set); - } - metric::metric_value proxy; - populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, - values_for_scaling); - break; - } - case constants::Error: - { - typedef model::metrics::error_metric metric_t; - typedef model::metric_base::metric_set metric_set_t; - const metric_set_t& metric_set = metrics.get_set(); - metric::metric_value proxy; - populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, - values_for_scaling); - break; - } - default: - INTEROP_THROW( model::invalid_metric_type, "Unsupported metric type: " << constants::to_string(type)); - }; - - if(!values_for_scaling.empty()) - { - std::sort(values_for_scaling.begin(), values_for_scaling.end()); - // TODO: Put this back - /* - const float lower = util::percentile_sorted(values_for_scaling.begin(), values_for_scaling.end(), - 25); - const float upper = util::percentile_sorted(values_for_scaling.begin(), values_for_scaling.end(), - 75);*/ - const float lower = values_for_scaling[size_t(0.25*values_for_scaling.size())]; - const float upper = values_for_scaling[size_t(0.75*values_for_scaling.size())]; - data.set_range(std::max(lower - 2 * (upper - lower), values_for_scaling[0]), - std::min(values_for_scaling.back(), upper + 2 * (upper - lower))); - } - else data.set_range(0,0); - if(type == constants::ErrorRate) data.set_range(0, std::min(5.0f, data.saxis().max())); - - std::string title = metrics.run_info().flowcell().barcode(); - if(title != "") title += " "; - title += utils::to_description(type); - data.set_title(title); + model::index_out_of_bounds_exception); - std::string subtitle; - if(metrics.run_info().flowcell().surface_count()>1) - subtitle += options.surface_description() + " "; - subtitle += options.cycle_description(); - if(logic::utils::is_channel_metric(type)) - subtitle += " " + options.channel_description(metrics.run_info().channels()); - if(logic::utils::is_base_metric(type)) - subtitle += " " + options.base_description(); - if(logic::utils::is_read_metric(type)) - subtitle += " " + options.read_description(); - data.set_subtitle(subtitle); - data.set_label(utils::to_description(type)); - } /** Plot a flowcell map * * @ingroup plot_logic @@ -200,36 +43,35 @@ namespace illumina { namespace interop { namespace logic { namespace plot { * @param metric_name specific metric value to plot by cycle * @param options options to filter the data * @param data output flowcell map + * @param buffer preallocated memory for data + * @param tile_buffer preallocated memory for tile ids */ - inline void plot_flowcell_map(model::metrics::run_metrics& metrics, + void plot_flowcell_map(model::metrics::run_metrics& metrics, const std::string& metric_name, const model::plot::filter_options& options, - model::plot::flowcell_data& data) + model::plot::flowcell_data& data, + float* buffer=0, + ::uint32_t* tile_buffer=0) throw(model::invalid_filter_option, model::invalid_metric_type, - model::index_out_of_bounds_exception) - { - const constants::metric_type type = constants::parse(metric_name); - if(type == constants::UnknownMetricType) - INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); - plot_flowcell_map(metrics, type, options, data); - } + model::index_out_of_bounds_exception); + /** List metric types available for flowcell + * + * @param types destination vector to fill with metric types + */ + void list_flowcell_metrics(std::vector& types); /** List metric type names available for flowcell * * @param names destination vector to fill with metric type names */ - inline void list_flowcell_metrics(std::vector& names) - { - std::vector types; - constants::list_enums(types); - names.clear(); - names.reserve(types.size()); - for(size_t i=0;i& names); + /** Calculate the required buffer size + * + * @param metrics run metrics + * @param options options to filter the data + */ + size_t calculate_flowcell_buffer_size(const model::metrics::run_metrics& metrics, + const model::plot::filter_options& options); }}}} diff --git a/interop/logic/plot/plot_qscore_heatmap.h b/interop/logic/plot/plot_qscore_heatmap.h index 88c9b731c..c35735197 100644 --- a/interop/logic/plot/plot_qscore_heatmap.h +++ b/interop/logic/plot/plot_qscore_heatmap.h @@ -7,175 +7,38 @@ */ #pragma once -#include "interop/util/statistics.h" -#include "interop/constants/enums.h" -#include "interop/model/model_exceptions.h" #include "interop/model/run_metrics.h" #include "interop/model/plot/filter_options.h" -#include "interop/model/plot/series.h" -#include "interop/model/plot/bar_point.h" #include "interop/model/plot/heatmap_data.h" -#include "interop/logic/metric/q_metric.h" -namespace illumina { namespace interop { namespace logic { namespace plot { - - - /** Populate the q-score heat map based on the filter options - * - * @param beg iterator to start of q-metric collection - * @param end iterator to end of q-metric collection - * @param bins q-score bins - * @param options filter for metric records - * @param data q-score heatmap - */ - template - void populate_heatmap_from_compressed(I beg, - I end, - const std::vector& bins, - const model::plot::filter_options &options, - model::plot::heatmap_data& data) - { - for (;beg != end;++beg) - { - if( !options.valid_tile(*beg) ) continue; - for(size_t bin =0;bin < bins.size();++bin) - data(beg->cycle()-1, bins[bin].value()-1) += beg->qscore_hist(bin); - } - } - /** Populate the q-score heatmap based on the filter options - * - * @param beg iterator to start of q-metric collection - * @param end iterator to end of q-metric collection - * @param options filter for metric records - * @param data q-score heatmap - */ - template - void populate_heatmap_from_uncompressed(I beg, - I end, - const model::plot::filter_options &options, - model::plot::heatmap_data& data) - { - for (;beg != end;++beg) - { - if( !options.valid_tile(*beg) ) continue; - - for(size_t bin =0;bin < beg->size();++bin) - data(beg->cycle()-1, bin) += beg->qscore_hist(bin); - } - } - /** Normalize the heat map to a percent - * - * @param data output heat map data - */ - inline void normalize_heatmap(model::plot::heatmap_data& data) - { - float max_value = 0; - for(size_t r=0;r - void remap_to_bins(I beg, I end, const size_t max_cycle, model::plot::heatmap_data& data) - { - for(;beg != end;++beg) - { - for(size_t bin = std::max(0, beg->lower()-1), upper=beg->upper();bin < upper;++bin) - { - for(size_t cycle = 0;cycle < max_cycle;++cycle) - { - data(cycle, bin) = data(cycle, beg->value()-1); - } - } - } - } - /** Plot a heat map of q-scores - * - * @param metrics q-metrics (full or by lane) - * @param options options to filter the data - * @param data output heat map datall - */ - template - void populate_heatmap(const model::metric_base::metric_set& metric_set, - const model::plot::filter_options& options, - model::plot::heatmap_data& data) - { - const size_t max_q_val = logic::metric::max_qval(metric_set); - const size_t max_cycle = metric_set.max_cycle(); - data.resize(max_cycle, max_q_val); - INTEROP_ASSERT(data.row_count() > 0); - INTEROP_ASSERTMSG(data.column_count() > 0, max_q_val << ", " << metric_set.size() << ", " << metric_set.bin_count() << ", " << metric::is_compressed(metric_set) << ", " << metric_set.bins().back().upper()); - const bool is_compressed = logic::metric::is_compressed(metric_set); - if(is_compressed) - populate_heatmap_from_compressed(metric_set.begin(), - metric_set.end(), - metric_set.bins(), - options, - data); - else - populate_heatmap_from_uncompressed(metric_set.begin(), - metric_set.end(), - options, - data); - normalize_heatmap(data); - remap_to_bins(metric_set.bins().begin(), - metric_set.bins().end(), - max_cycle, - data); - } +namespace illumina { namespace interop { namespace logic { namespace plot +{ /** Plot a heat map of q-scores * * @ingroup plot_logic * @param metrics run metrics * @param options options to filter the data * @param data output heat map data + * @param buffer optional buffer of preallocated memory (for SWIG) */ - inline void plot_qscore_heatmap(model::metrics::run_metrics& metrics, + void plot_qscore_heatmap(model::metrics::run_metrics& metrics, const model::plot::filter_options& options, - model::plot::heatmap_data& data) - throw(model::index_out_of_bounds_exception, model::invalid_filter_option) - { - options.validate(constants::QScore, metrics.run_info()); - data.clear(); - if(options.is_specific_surface()) - { - typedef model::metrics::q_metric metric_t; - if (metrics.get_set().size() == 0)return; - populate_heatmap(metrics.get_set(), options, data); - } - else - { - typedef model::metrics::q_by_lane_metric metric_t; - if(0 == metrics.get_set().size()) - logic::metric::create_q_metrics_by_lane(metrics.get_set(), - metrics.get_set()); - if (metrics.get_set().size() == 0)return; - populate_heatmap(metrics.get_set(), options, data); - } - - data.set_xrange(0, static_cast(data.row_count())); - data.set_yrange(0, static_cast(data.column_count())); - - data.set_xlabel("Cycle"); - data.set_ylabel("Q Score"); - - std::string title = metrics.run_info().flowcell().barcode(); - if(title != "") title += " "; - title += options.lane_description(); - if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) - title += " " + options.surface_description(); - data.set_title(title); - } + model::plot::heatmap_data& data, + float* buffer=0) + throw(model::index_out_of_bounds_exception, + model::invalid_filter_option); + /** Count number of rows for the heat map + * + * @param metrics run metrics + * @return number of rows + */ + size_t count_rows_for_heatmap(const model::metrics::run_metrics& metrics); + /** Count number of columns for the heat map + * + * @param metrics run metrics + * @return number of columns + */ + size_t count_columns_for_heatmap(const model::metrics::run_metrics& metrics); }}}} diff --git a/interop/logic/plot/plot_qscore_histogram.h b/interop/logic/plot/plot_qscore_histogram.h index 42ab3d1ee..e47244252 100644 --- a/interop/logic/plot/plot_qscore_histogram.h +++ b/interop/logic/plot/plot_qscore_histogram.h @@ -5,144 +5,15 @@ * @version 1.0 * @copyright GNU Public License. */ -#pragma once -#include "interop/util/statistics.h" -#include "interop/constants/enums.h" -#include "interop/model/model_exceptions.h" #include "interop/model/run_metrics.h" #include "interop/model/plot/filter_options.h" -#include "interop/model/plot/series.h" #include "interop/model/plot/bar_point.h" #include "interop/logic/plot/plot_data.h" namespace illumina { namespace interop { namespace logic { namespace plot { - /** Populate the q-score histogram based on the filter options - * - * @param beg iterator to start of q-metric collection - * @param end iterator to end of q-metric collection - * @param options filter for metric records - * @param first_cycle first cycle to keep - * @param last_cycle last cycle to keep - * @param histogram q-score histogram - */ - template - void populate_distribution(I beg, - I end, - const model::plot::filter_options &options, - const size_t first_cycle, - const size_t last_cycle, - std::vector& histogram) - { - if(beg == end) return; - histogram.resize(beg->size(), 0); - for (;beg != end;++beg) - { - if( !options.valid_tile(*beg) || beg->cycle() < first_cycle || beg->cycle() > last_cycle) continue; - beg->accumulate_into(histogram); - } - } - /** Scale the histogram if necessary and provide the scale label - * - * @param histogram q-score histogram - * @return label of the scale - */ - inline std::string scale_histogram(std::vector& histogram) - { - float max_height = 0; - for(size_t i=0;i(options.cycle())); - return last_cycle; - } - /** Plot an unbinned histogram - * - * @param histogram q-score histogram - * @param points collection of points where x is lane number and y is the candle stick metric values - * @return max x-value - */ - template - float plot_unbinned_histogram(const std::vector& histogram, - model::plot::data_point_collection& points) - { - points.resize(histogram.size()); - size_t point_index = 0; - for(size_t i=0;i(i+1), histogram[i]); - ++point_index; - } - points.resize(point_index); - if(point_index == 0) return 0; - return static_cast(points[point_index-1].x()+1); - } - /** Plot a binned histogram - * - * @param beg iterator to start of the histogram bins - * @param end iterator to end of the histogram bins - * @param histogram q-score histogram - * @param points collection of points where x is lane number and y is the candle stick metric values - * @return max x-value - */ - template - float plot_binned_histogram(I beg, - I end, - const std::vector& histogram, - model::plot::data_point_collection& points) - { - float max_x_value = 0; - points.resize(std::distance(beg, end)); - size_t point_index = 0; - if(points.size() == histogram.size()) //Compressed - { - for (size_t i=0; beg != end; ++beg, ++i) - { - INTEROP_ASSERT(i < histogram.size()); - if(histogram[i] == 0)continue; - points[point_index].set(beg->lower(), histogram[i], static_cast(beg->upper()-beg->lower()+1)); - max_x_value = std::max(max_x_value, points[point_index].x()+points[point_index].width()); - ++point_index; - } - } - else // Uncompressed - { - for (size_t i=0; beg != end; ++beg, ++i) - { - const size_t bin = static_cast(beg->value())-1; - INTEROP_ASSERTMSG(bin < histogram.size(), bin << " < " << histogram.size()); - if(histogram[bin] == 0)continue; - points[point_index].set(beg->lower(), histogram[bin], static_cast(beg->upper()-beg->lower()+1)); - max_x_value = std::max(max_x_value, points[point_index].x()+points[point_index].width()); - ++point_index; - } - } - points.resize(point_index); - return max_x_value; - } - /** Plot a histogram of q-scores * * @ingroup plot_logic @@ -151,131 +22,12 @@ namespace illumina { namespace interop { namespace logic { namespace plot { * @param data output plot data * @param boundary index of bin to create the boundary sub plots (0 means do nothing) */ - template void plot_qscore_histogram(model::metrics::run_metrics& metrics, const model::plot::filter_options& options, - model::plot::plot_data& data, + model::plot::plot_data& data, const size_t boundary=0) throw( model::invalid_read_exception, model::index_out_of_bounds_exception, - model::invalid_filter_option) - { - options.validate(constants::QScore, metrics.run_info()); - data.clear(); - const size_t first_cycle = options.all_reads() ? 1 : metrics.run_info().read(options.read()).first_cycle(); - - data.push_back(model::plot::series("Q Score", "Blue", model::plot::series::Bar)); - if(boundary>0) - { - data.push_back(model::plot::series("Q Score", "DarkGreen", model::plot::series::Bar)); - data.push_back(model::plot::series("Threshold(>=Q30)", "Green", model::plot::series::Line)); - } - - //"DarkGreen" - - for(size_t i=0;i histogram; - float max_x_value; - - if(options.is_specific_surface()) - { - typedef model::metrics::q_metric metric_t; - const size_t last_cycle = get_last_filtered_cycle(metrics.run_info(), - options, - metrics.get_set().max_cycle()); - if(metrics.get_set().size() == 0) - { - //data.clear(); // TODO: Add this back? - return; - } - populate_distribution( - metrics.get_set().begin(), - metrics.get_set().end(), - options, - first_cycle, - last_cycle, - histogram); - axis_scale = scale_histogram(histogram); - if(!metrics.get_set().bins().empty()) - max_x_value=plot_binned_histogram(metrics.get_set().bins().begin(), - metrics.get_set().bins().end(), - histogram, - data[0]); - else max_x_value=plot_unbinned_histogram(histogram, data[0]); - } - else - { - typedef model::metrics::q_by_lane_metric metric_t; - if(0 == metrics.get_set().size()) - logic::metric::create_q_metrics_by_lane(metrics.get_set(), - metrics.get_set()); - if(0 == metrics.get_set().size()) - { - //data.clear(); // TODO: Add this back? - return; - } - const size_t last_cycle = get_last_filtered_cycle(metrics.run_info(), - options, - metrics.get_set().max_cycle()); - INTEROP_ASSERT(0 != metrics.get_set().size()); - populate_distribution( - metrics.get_set().begin(), - metrics.get_set().end(), - options, - first_cycle, - last_cycle, - histogram); - axis_scale = scale_histogram(histogram); - if(!metrics.get_set().bins().empty()) - max_x_value=plot_binned_histogram(metrics.get_set().bins().begin(), - metrics.get_set().bins().end(), - histogram, - data[0]); - else max_x_value=plot_unbinned_histogram(histogram, data[0]); - } - - // split into two series, < Q30 and Q30 <= - if(data.size() > 1) - { - size_t gt_eq_boundary_count = 0; - data[1].resize(data[0].size()); - for (size_t i = 0; i < data[0].size(); ++i) - { - if (data[0][i].x() >= boundary) - { - data[1][gt_eq_boundary_count] = data[0][i]; - ++gt_eq_boundary_count; - } - } - data[0].resize(data[0].size() - gt_eq_boundary_count); - data[1].resize(gt_eq_boundary_count); - } - - - auto_scale_y(data, false); - data.set_xrange(1, std::max(1.0f, max_x_value)*1.1f); - if(data.size() > 2) - { - data[2].resize(2); - data[2][0].set(static_cast(boundary), data.y_axis().min()); - data[2][1].set(static_cast(boundary), data.y_axis().max()); - } - - data.set_xlabel("Q Score"); - data.set_ylabel("Total ("+axis_scale+")"); - - std::string title = metrics.run_info().flowcell().barcode(); - if(title != "") title += " "; - title += options.lane_description(); - if(options.is_specific_read()) - title += " " + options.read_description(); - if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) - title += " " + options.surface_description(); - data.set_title(title); - } - + model::invalid_filter_option); }}}} diff --git a/interop/logic/plot/plot_sample_qc.h b/interop/logic/plot/plot_sample_qc.h index beee75742..932533b00 100644 --- a/interop/logic/plot/plot_sample_qc.h +++ b/interop/logic/plot/plot_sample_qc.h @@ -6,75 +6,14 @@ * @copyright GNU Public License. */ #pragma once -#include -#include -#include "interop/util/statistics.h" #include "interop/constants/enums.h" -#include "interop/logic/utils/enums.h" -#include "interop/model/model_exceptions.h" #include "interop/model/run_metrics.h" #include "interop/model/plot/bar_point.h" #include "interop/logic/plot/plot_data.h" -namespace illumina { namespace interop { namespace logic { namespace plot { +namespace illumina { namespace interop { namespace logic { namespace plot +{ - /** Populate reads identified versus the index - * - * @param index_metrics set of metric records - * @param tile_metrics source collection of tile metrics - * @param proxy functor that takes a metric record and returns a metric value - * @param options filter for metric records - * @param type type of metric to extract using the proxy functor - * @param points collection of points where x is lane number and y is the candle stick metric values - */ - template - float populate_reads_identified(const model::metric_base::metric_set& index_metrics, - const model::metric_base::metric_set& tile_metrics, - const size_t lane, - model::plot::data_point_collection& points) - { - typedef model::metric_base::metric_set index_metric_set_t; - typedef std::map index_count_map_t; - typedef typename index_count_map_t::iterator map_iterator; - typedef typename index_count_map_t::const_iterator const_map_iterator; - typedef typename model::metrics::index_metric::const_iterator const_index_iterator; - - index_count_map_t index_count_map; - float pf_cluster_count_total = 0; - for(typename index_metric_set_t::const_iterator b = index_metrics.begin(), e = index_metrics.end();b != e;++b) - { - if(lane != b->lane()) continue; - try - { - const model::metrics::tile_metric &tile_metric = tile_metrics.get_metric(b->lane(), b->tile()); - pf_cluster_count_total += tile_metric.cluster_count_pf(); - for(const_index_iterator ib = b->indices().begin(), ie = b->indices().end();ib != ie;++ib) - { - map_iterator found_index = index_count_map.find(ib->index_seq()); - if(found_index == index_count_map.end()) index_count_map[ib->index_seq()] = ib->count(); - else found_index->second += ib->count(); - } - } - catch(const model::index_out_of_bounds_exception&){continue;} // TODO: check better? - } - points.resize(index_count_map.size()); - float max_height=0; - std::vector heights; - heights.reserve(index_count_map.size()); - size_t i=0; - std::vector keys; - keys.reserve(index_count_map.size()); - for(const_map_iterator b = index_count_map.begin(), e = index_count_map.end();b != e;++b) - keys.push_back(b->first); - std::stable_sort(keys.begin(), keys.end()); - for(typename std::vector::const_iterator b = keys.begin(), e = keys.end();b != e;++b,++i) - { - const float height = (pf_cluster_count_total==0) ? 0 : index_count_map[*b] / pf_cluster_count_total * 100.0f; - points[i].set(i+1.0f, height, 1.0f); - max_height = std::max(max_height, height); - } - return max_height; - } /** Plot reads identified versus index * @@ -83,41 +22,10 @@ namespace illumina { namespace interop { namespace logic { namespace plot { * @param lane lane number * @param data output plot data */ - inline void plot_sample_qc(const model::metrics::run_metrics& metrics, + void plot_sample_qc(const model::metrics::run_metrics& metrics, const size_t lane, model::plot::plot_data& data) - throw(model::index_out_of_bounds_exception) - { - typedef model::plot::series bar_series_t; - data.set_xlabel("Index Number"); - data.set_ylabel("% Reads Identified (PF)"); - data.assign(1, bar_series_t("% reads", "Green", bar_series_t::Bar)); - data[0].add_option(constants::to_string(constants::Centered)); - - if(metrics.get_set().size() == 0) - { - if(metrics.run_info().is_indexed()) - { - data.set_range(data.x_axis().min(), - 1, - data.y_axis().min(), - 5); - } - //data.clear(); // TODO: Remove below and uncomment this line - - return; - } - - const float max_height = populate_reads_identified(metrics.get_set(), - metrics.get_set(), - lane, - data[0]); - auto_scale(data); - data.set_range(data.x_axis().min(), - static_cast(data[0].size()+1), - data.y_axis().min(), - roundf(max_height+5)); - } + throw(model::index_out_of_bounds_exception); }}}} diff --git a/interop/logic/summary/cycle_state_summary.h b/interop/logic/summary/cycle_state_summary.h index 26f31c5bb..3de044271 100644 --- a/interop/logic/summary/cycle_state_summary.h +++ b/interop/logic/summary/cycle_state_summary.h @@ -6,109 +6,116 @@ * @copyright GNU Public License. */ #pragma once + #include "interop/logic/summary/map_cycle_to_read.h" #include "interop/model/metric_base/metric_set.h" #include "interop/model/metrics/tile_metric.h" -namespace illumina { namespace interop { namespace logic { namespace summary { +namespace illumina { namespace interop { namespace logic { namespace summary +{ -/** Define a member function of cycle_state_summary that sets a cycle state */ -typedef void (model::summary::cycle_state_summary::*set_cycle_state_func_t )(const model::run::cycle_range&); -/** Vector of vectors of cycle_range objects */ -typedef std::vector > cycle_range_vector2d_t; + /** Define a member function of cycle_state_summary that sets a cycle state */ + typedef void (model::summary::cycle_state_summary::*set_cycle_state_func_t )(const model::run::cycle_range &); + /** Vector of vectors of cycle_range objects */ + typedef std::vector > cycle_range_vector2d_t; -/** Summarize the cycle state for a particular metric - * - * @param beg iterator to start of a collection of error metrics - * @param end iterator to end of a collection of error metrics - * @param cycle_to_read map cycle to the read number and cycle within read number - * @param pointer to function that sets a cycle state - * @param run destination run summary - */ -template -void summarize_cycle_state(const model::metric_base::metric_set& tile_metrics, - const model::metric_base::metric_set& cycle_metrics, - const read_cycle_vector_t& cycle_to_read, - set_cycle_state_func_t set_cycle_state_fun, - model::summary::run_summary &run) throw( model::index_out_of_bounds_exception ) -{ - typedef model::run::cycle_range cycle_range; - typedef typename model::metric_base::metric_set::const_iterator const_tile_iterator; - typedef typename model::metric_base::metric_set::const_iterator const_metric_iterator; - cycle_range_vector2d_t summary_by_lane_read(run.size(), std::vector< cycle_range >(run.lane_count())); - typedef model::metrics::tile_metric::id_t id_t; - typedef std::map< id_t, cycle_range > cycle_range_tile_t; - typedef std::map< id_t, size_t > max_tile_map_t; - typedef typename max_tile_map_t::const_iterator const_max_tile_iterator; - typedef std::vector< cycle_range_tile_t > cycle_range_by_read_tile_t; - cycle_range_by_read_tile_t tmp (summary_by_lane_read.size()); - max_tile_map_t tmp_by_tile; - cycle_range overall_cycle_state; - for(const_metric_iterator cycle_metric_it = cycle_metrics.begin(), cycle_metric_end=cycle_metrics.end(); - cycle_metric_it != cycle_metric_end;++cycle_metric_it) + /** Summarize the cycle state for a particular metric + * + * @param tile_metrics tile metric set + * @param cycle_metrics a cycle based metric set + * @param cycle_to_read map between the current cycle and read information + * @param set_cycle_state_fun callback to set the cycle state + * @param run run summary + */ + template + void summarize_cycle_state(const model::metric_base::metric_set &tile_metrics, + const model::metric_base::metric_set &cycle_metrics, + const read_cycle_vector_t &cycle_to_read, + set_cycle_state_func_t set_cycle_state_fun, + model::summary::run_summary &run) throw(model::index_out_of_bounds_exception) { - INTEROP_ASSERT(cycle_metric_it->cycle() > 0); - INTEROP_ASSERT((cycle_metric_it->cycle()-1) < cycle_to_read.size()); + typedef model::run::cycle_range cycle_range; + typedef typename model::metric_base::metric_set::const_iterator const_tile_iterator; + typedef typename model::metric_base::metric_set::const_iterator const_metric_iterator; + cycle_range_vector2d_t summary_by_lane_read(run.size(), std::vector(run.lane_count())); - if((cycle_metric_it->cycle()-1) >= cycle_to_read.size()) - throw model::index_out_of_bounds_exception("Cycle exceeds total cycles from Reads in the RunInfo.xml"); - const read_cycle& read = cycle_to_read[cycle_metric_it->cycle()-1]; - if(read.number == 0) continue; - INTEROP_ASSERT((read.number-1) < tmp.size()); - const id_t id = model::metric_base::base_metric::id(cycle_metric_it->lane(), cycle_metric_it->tile()); - tmp[read.number-1][id].update(cycle_metric_it->cycle()); - typename max_tile_map_t::iterator it = tmp_by_tile.find(id); - if( it == tmp_by_tile.end() ) tmp_by_tile[id] = cycle_metric_it->cycle(); - else it->second = std::max(static_cast(cycle_metric_it->cycle()), it->second); - } + typedef model::metrics::tile_metric::id_t id_t; + typedef std::map cycle_range_tile_t; + typedef std::map max_tile_map_t; + typedef typename max_tile_map_t::const_iterator const_max_tile_iterator; + typedef std::vector cycle_range_by_read_tile_t; + cycle_range_by_read_tile_t tmp(summary_by_lane_read.size()); + max_tile_map_t tmp_by_tile; + cycle_range overall_cycle_state; + for (const_metric_iterator cycle_metric_it = cycle_metrics.begin(), cycle_metric_end = cycle_metrics.end(); + cycle_metric_it != cycle_metric_end; ++cycle_metric_it) + { + INTEROP_ASSERT(cycle_metric_it->cycle() > 0); + INTEROP_ASSERT((cycle_metric_it->cycle() - 1) < cycle_to_read.size()); - // Tile exists, but nothing was written out for that metric on any cycle - for(const_tile_iterator tile_it = tile_metrics.begin(), tile_end=tile_metrics.end();tile_it != tile_end;++tile_it) - { - size_t cycle_for_tile = 0; - const id_t id = model::metric_base::base_metric::id(tile_it->lane(), tile_it->tile()); - for(size_t read_index = 0; read_index < tmp.size(); ++read_index) + if ((cycle_metric_it->cycle() - 1) >= cycle_to_read.size()) + throw model::index_out_of_bounds_exception("Cycle exceeds total cycles from Reads in the RunInfo.xml"); + const read_cycle &read = cycle_to_read[cycle_metric_it->cycle() - 1]; + if (read.number == 0) continue; + INTEROP_ASSERT((read.number - 1) < tmp.size()); + const id_t id = model::metric_base::base_metric::create_id(cycle_metric_it->lane(), cycle_metric_it->tile()); + tmp[read.number - 1][id].update(cycle_metric_it->cycle()); + typename max_tile_map_t::iterator it = tmp_by_tile.find(id); + if (it == tmp_by_tile.end()) tmp_by_tile[id] = cycle_metric_it->cycle(); + else it->second = std::max(static_cast(cycle_metric_it->cycle()), it->second); + } + + // Tile exists, but nothing was written out for that metric on any cycle + for (const_tile_iterator tile_it = tile_metrics.begin(), tile_end = tile_metrics.end(); + tile_it != tile_end; ++tile_it) { - if (tmp[read_index].find(tile_it->id()) == tmp[read_index].end()) + size_t cycle_for_tile = 0; + const id_t id = model::metric_base::base_metric::create_id(tile_it->lane(), tile_it->tile()); + for (size_t read_index = 0; read_index < tmp.size(); ++read_index) { - tmp[read_index][id].update(run[read_index].read().first_cycle()-1); - tmp_by_tile[id] = 0; - } - else - { - cycle_for_tile = tmp[read_index][id].last_cycle(); + if (tmp[read_index].find(tile_it->id()) == tmp[read_index].end()) + { + tmp[read_index][id].update(run[read_index].read().first_cycle() - 1); + tmp_by_tile[id] = 0; + } + else + { + cycle_for_tile = tmp[read_index][id].last_cycle(); + } } + tmp_by_tile[id] = cycle_for_tile; } - tmp_by_tile[id] = cycle_for_tile; - } - for(size_t read=0;read(model::metric_base::base_metric::lane_from_id(it->first)-1); - INTEROP_ASSERT(lane < summary_by_lane_read[read].size()); - summary_by_lane_read[read][lane].update(it->second.last_cycle()); + INTEROP_ASSERT(read < summary_by_lane_read.size()); + for (typename cycle_range_tile_t::const_iterator it = tmp[read].begin(), end = tmp[read].end(); + it != end; ++it) + { + const size_t lane = static_cast(model::metric_base::base_metric::lane_from_id(it->first) - 1); + INTEROP_ASSERT(lane < summary_by_lane_read[read].size()); + summary_by_lane_read[read][lane].update(it->second.last_cycle()); + } } - } - for(size_t read=0;readsecond); + (run.cycle_state().*set_cycle_state_fun)(overall_cycle_state); } - for(const_max_tile_iterator range_it = tmp_by_tile.begin(), range_end = tmp_by_tile.end();range_it != range_end; - ++range_it) - overall_cycle_state.update(range_it->second); - (run.cycle_state().*set_cycle_state_fun)(overall_cycle_state); -} }}}} diff --git a/interop/logic/summary/error_summary.h b/interop/logic/summary/error_summary.h index e942f4ab3..463c9c5f9 100644 --- a/interop/logic/summary/error_summary.h +++ b/interop/logic/summary/error_summary.h @@ -18,176 +18,181 @@ #include "interop/model/summary/run_summary.h" -namespace illumina { namespace interop { namespace logic { namespace summary { - -/** Cache errors for all tiles up to a give max cycle - * - * This function only includes errors from useable cycles (not the last cycle) to up the given max cycle. - * - * @param beg iterator to start of a collection of error metrics - * @param end iterator to end of a collection of error metrics - * @param max_cycle maximum cycle to take - * @param cycle_to_read map that takes a cycle and returns the read-number cycle-in-read pair - * @param summary_by_lane_read destination cache by read then by lane a collection of errors - */ - -template -void cache_error_by_lane_read(I beg, - I end, - const size_t max_cycle, - const std::vector& cycle_to_read, - summary_by_lane_read& summary_by_lane_read) - throw( model::index_out_of_bounds_exception ) +namespace illumina { namespace interop { namespace logic { namespace summary { - typedef std::vector cycle_vector_t; - typedef std::pair key_t; - typedef std::pair value_t; - typedef std::map< key_t, value_t> error_tile_t; - typedef std::vector< error_tile_t > error_by_read_tile_t; - error_by_read_tile_t tmp (summary_by_lane_read.size()); - cycle_vector_t max_error_cycle(summary_by_lane_read.size(), 0); - for(;beg != end;++beg) - { - INTEROP_ASSERT(beg->cycle() > 0); - INTEROP_ASSERT((beg->cycle()-1) < cycle_to_read.size()); - if((beg->cycle()-1) >=cycle_to_read.size()) - throw model::index_out_of_bounds_exception("Cycle exceeds total cycles from Reads in the RunInfo.xml"); - const read_cycle& read = cycle_to_read[beg->cycle()-1]; - if(read.cycle_within_read > max_cycle || read.is_last_cycle_in_read) continue; - key_t key = std::make_pair(beg->lane(), beg->tile()); - const size_t read_number = read.number-1; - max_error_cycle[read_number] = std::max(max_error_cycle[read_number], static_cast(read.cycle_within_read)); - INTEROP_ASSERTMSG(read_number < tmp.size(), read.number << " " << read.cycle_within_read << ", " << beg->cycle()); - if(tmp[read_number].find(key) == tmp[read_number].end()) - tmp[read_number][key] = std::make_pair(0.0f, 0.0f); - tmp[read_number][key].first+=beg->errorRate(); - tmp[read_number][key].second+=1; - } - for(size_t read=0;read + void cache_error_by_lane_read(I beg, + I end, + const size_t max_cycle, + const std::vector &cycle_to_read, + summary_by_lane_read &summary_by_lane_read) + throw(model::index_out_of_bounds_exception) { - for(typename error_tile_t::const_iterator ebeg = tmp[read].begin(), eend = tmp[read].end();ebeg != eend;++ebeg) + typedef std::vector cycle_vector_t; + typedef std::pair key_t; + typedef std::pair value_t; + typedef std::map error_tile_t; + typedef std::vector error_by_read_tile_t; + error_by_read_tile_t tmp(summary_by_lane_read.size()); + cycle_vector_t max_error_cycle(summary_by_lane_read.size(), 0); + for (; beg != end; ++beg) { - INTEROP_ASSERT(read < summary_by_lane_read.read_count()); - const size_t lane = ebeg->first.first-1; - if(lane >= summary_by_lane_read.lane_count()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Lane exceeds number of lanes in RunInfo.xml"); - if(max_cycle < std::numeric_limits::max() && max_error_cycle[read] < max_cycle) - summary_by_lane_read(read, lane).push_back(0); - else summary_by_lane_read(read, lane).push_back(divide(ebeg->second.first,ebeg->second.second)); + INTEROP_ASSERT(beg->cycle() > 0); + INTEROP_ASSERT((beg->cycle() - 1) < cycle_to_read.size()); + if ((beg->cycle() - 1) >= cycle_to_read.size()) + throw model::index_out_of_bounds_exception("Cycle exceeds total cycles from Reads in the RunInfo.xml"); + const read_cycle &read = cycle_to_read[beg->cycle() - 1]; + if (read.cycle_within_read > max_cycle || read.is_last_cycle_in_read) continue; + key_t key = std::make_pair(beg->lane(), beg->tile()); + const size_t read_number = read.number - 1; + max_error_cycle[read_number] = std::max(max_error_cycle[read_number], + static_cast(read.cycle_within_read)); + INTEROP_ASSERTMSG(read_number < tmp.size(), + read.number << " " << read.cycle_within_read << ", " << beg->cycle()); + if (tmp[read_number].find(key) == tmp[read_number].end()) + tmp[read_number][key] = std::make_pair(0.0f, 0.0f); + tmp[read_number][key].first += beg->error_rate(); + tmp[read_number][key].second += 1; + } + for (size_t read = 0; read < tmp.size(); ++read) + { + for (typename error_tile_t::const_iterator ebeg = tmp[read].begin(), eend = tmp[read].end(); + ebeg != eend; ++ebeg) + { + INTEROP_ASSERT(read < summary_by_lane_read.read_count()); + const size_t lane = ebeg->first.first - 1; + if (lane >= summary_by_lane_read.lane_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Lane exceeds number of lanes in RunInfo.xml"); + if (max_cycle < std::numeric_limits::max() && max_error_cycle[read] < max_cycle) + summary_by_lane_read(read, lane).push_back(0); + else summary_by_lane_read(read, lane).push_back(divide(ebeg->second.first, ebeg->second.second)); + } } } -} -/** Calculate summary statistics for each collection of metrics organized by read and lane - * - * @param summary_by_lane_read source cache by read then by lane a collection of errors - * @param run destination run summary - * @param func member function pointer to metric - */ -inline void error_summary_from_cache(summary_by_lane_read& summary_by_lane_read, - model::summary::run_summary& run, - model::summary::metric_stat& (model::summary::lane_summary::*func )()) -{ - for(size_t read=0;read &summary_by_lane_read, + model::summary::run_summary &run, + void (model::summary::lane_summary::*func )(const model::summary::metric_stat&)) { - INTEROP_ASSERT(read < summary_by_lane_read.read_count()); - INTEROP_ASSERT(read < run.size()); - for (size_t lane = 0; lane < run[read].size(); ++lane) + for (size_t read = 0; read < run.size(); ++read) { - INTEROP_ASSERT(lane < run[read].size()); - INTEROP_ASSERT(lane < summary_by_lane_read.lane_count()); - summarize(summary_by_lane_read(read,lane).begin(), - summary_by_lane_read(read,lane).end(), - (run[read][lane].*func)()); + INTEROP_ASSERT(read < summary_by_lane_read.read_count()); + INTEROP_ASSERT(read < run.size()); + for (size_t lane = 0; lane < run[read].size(); ++lane) + { + INTEROP_ASSERT(lane < run[read].size()); + INTEROP_ASSERT(lane < summary_by_lane_read.lane_count()); + model::summary::metric_stat stat; + summarize(summary_by_lane_read(read, lane).begin(), + summary_by_lane_read(read, lane).end(), + stat); + (run[read][lane].*func)(stat); + } } } -} -/** Summarize a collection error metrics - * - * @sa model::summary::lane_summary::error_rate - * @sa model::summary::lane_summary::error_rate_35 - * @sa model::summary::lane_summary::error_rate_50 - * @sa model::summary::lane_summary::error_rate_75 - * @sa model::summary::lane_summary::error_rate_100 - * - * @sa model::summary::read_summary::error_rate - * - * @sa model::summary::run_summary::error_rate - * - * @todo: Refactor this to make simpler - * - * @param beg iterator to start of a collection of error metrics - * @param end iterator to end of a collection of error metrics - * @param cycle_to_read map cycle to the read number and cycle within read number - * @param run destination run summary - */ -template -void summarize_error_metrics(I beg, - I end, - const read_cycle_vector_t& cycle_to_read, - model::summary::run_summary &run) throw( model::index_out_of_bounds_exception ) -{ - typedef summary_by_lane_read summary_by_lane_read_t; - typedef model::summary::metric_stat& (model::summary::lane_summary::*error_functor_t )(); - typedef std::pair cycle_functor_pair_t; - - if(beg == end) return; - if(run.size() == 0) return; - summary_by_lane_read_t temp(run, std::distance(beg, end)); - - cycle_functor_pair_t cycle_functor_pairs[] = { - cycle_functor_pair_t(35u, &model::summary::lane_summary::error_rate_35), - cycle_functor_pair_t(50u, &model::summary::lane_summary::error_rate_50), - cycle_functor_pair_t(75u, &model::summary::lane_summary::error_rate_75), - cycle_functor_pair_t(100u, &model::summary::lane_summary::error_rate_100), - }; - for(size_t i=0;i + void summarize_error_metrics(I beg, + I end, + const read_cycle_vector_t &cycle_to_read, + model::summary::run_summary &run) throw(model::index_out_of_bounds_exception) { - cache_error_by_lane_read(beg, end, cycle_functor_pairs[i].first, cycle_to_read, temp); - error_summary_from_cache(temp, run, cycle_functor_pairs[i].second); - temp.clear(); - } + typedef summary_by_lane_read summary_by_lane_read_t; + typedef void (model::summary::lane_summary::*error_functor_t )(const model::summary::metric_stat&); + typedef std::pair cycle_functor_pair_t; + if (beg == end) return; + if (run.size() == 0) return; + summary_by_lane_read_t temp(run, std::distance(beg, end)); - cache_error_by_lane_read(beg, end, std::numeric_limits::max(), cycle_to_read, temp); - - float error_rate = 0; - size_t total = 0; - float error_rate_nonindex = 0; - size_t total_nonindex = 0; - for(size_t read=0;read0) run[read].summary().error_rate(divide(error_rate_by_read,static_cast(total_by_read))); - error_rate += error_rate_by_read; - total += total_by_read; - // We keep track of the throughput for non-index reads - if(!run[read].read().is_index()) + + cache_error_by_lane_read(beg, end, std::numeric_limits::max(), cycle_to_read, temp); + + float error_rate = 0; + size_t total = 0; + float error_rate_nonindex = 0; + size_t total_nonindex = 0; + for (size_t read = 0; read < run.size(); ++read) { - error_rate_nonindex += error_rate_by_read; - total_nonindex += total_by_read; + INTEROP_ASSERT(read < run.size()); + float error_rate_by_read = 0; + size_t total_by_read = 0; + for (size_t lane = 0; lane < run[read].size(); ++lane) + { + INTEROP_ASSERT(lane < run[read].size()); + model::summary::metric_stat stat; + summarize(temp(read, lane).begin(), + temp(read, lane).end(), + stat); + run[read][lane].error_rate(stat); + error_rate_by_read += std::accumulate(temp(read, lane).begin(), + temp(read, lane).end(), + float(0)); + total_by_read += temp(read, lane).size(); + } + if (total_by_read > 0) + run[read].summary().error_rate(divide(error_rate_by_read, static_cast(total_by_read))); + error_rate += error_rate_by_read; + total += total_by_read; + + // We keep track of the throughput for non-index reads + if (!run[read].read().is_index()) + { + error_rate_nonindex += error_rate_by_read; + total_nonindex += total_by_read; + } } + run.nonindex_summary().error_rate(divide(error_rate_nonindex, static_cast(total_nonindex))); + run.total_summary().error_rate(divide(error_rate, static_cast(total))); } - run.nonindex_summary().error_rate(divide(error_rate_nonindex,static_cast(total_nonindex))); - run.total_summary().error_rate(divide(error_rate,static_cast(total))); -} -} -} -} -} +}}}} diff --git a/interop/logic/summary/extraction_summary.h b/interop/logic/summary/extraction_summary.h index 0d95de979..fd8fb0e82 100644 --- a/interop/logic/summary/extraction_summary.h +++ b/interop/logic/summary/extraction_summary.h @@ -67,11 +67,13 @@ namespace illumina { namespace interop { namespace logic { namespace summary INTEROP_ASSERT(read < run.size()); float first_cycle_intensity_by_read = 0; size_t total_by_read = 0; + model::summary::metric_stat first_cycle_intensity_stat; for (size_t lane = 0; lane < run[read].size(); ++lane) { INTEROP_ASSERT(lane < temp.lane_count()); INTEROP_ASSERT(lane < run[read].size()); - summarize(temp(read, lane).begin(), temp(read, lane).end(), run[read][lane].first_cycle_intensity()); + summarize(temp(read, lane).begin(), temp(read, lane).end(), first_cycle_intensity_stat); + run[read][lane].first_cycle_intensity(first_cycle_intensity_stat); first_cycle_intensity_by_read += std::accumulate(temp(read, lane).begin(), temp(read, lane).end(), size_t(0)); diff --git a/interop/logic/summary/index_summary.h b/interop/logic/summary/index_summary.h index bdc1a7981..f7dfccd72 100644 --- a/interop/logic/summary/index_summary.h +++ b/interop/logic/summary/index_summary.h @@ -7,124 +7,23 @@ */ #pragma once #include "interop/model/model_exceptions.h" -#include "interop/util/statistics.h" #include "interop/model/summary/index_flowcell_summary.h" #include "interop/model/run_metrics.h" -namespace illumina { namespace interop { namespace logic { namespace summary { +namespace illumina { namespace interop { namespace logic { namespace summary +{ - /** Summarize a index metrics for a specific lane - * - * @param beg iterator to start of source collection of index metrics - * @param end iterator to end of source collection of index metrics - * @param tile_metrics source collection of tile metrics - * @param lane lane number - * @param summary destination index flowcell summary - */ - template - inline void summarize_index_metrics(I beg, - I end, - const model::metric_base::metric_set& tile_metrics, - const size_t lane, - model::summary::index_lane_summary &summary) - throw(model::index_out_of_bounds_exception) - { - typedef typename model::summary::index_lane_summary::read_count_t read_count_t; - typedef typename model::metrics::index_metric::const_iterator const_index_iterator; - typedef model::summary::index_count_summary index_count_summary; - typedef std::map index_count_map_t; - typedef typename index_count_map_t::iterator map_iterator; - - index_count_map_t index_count_map; - size_t total_mapped_reads = 0; - read_count_t pf_cluster_count_total = 0; - read_count_t cluster_count_total = 0; - for(;beg != end;++beg) - { - if(beg->lane() != lane) continue; - try - { - const model::metrics::tile_metric &tile_metric = tile_metrics.get_metric(beg->lane(), beg->tile()); - pf_cluster_count_total += static_cast(tile_metric.cluster_count_pf()); - cluster_count_total += static_cast(tile_metric.cluster_count()); - - for(const_index_iterator ib = beg->indices().begin(), ie = beg->indices().end();ib != ie;++ib) - { - map_iterator found_index = index_count_map.find(ib->index_seq()); - if(found_index == index_count_map.end()) - { - index_count_map[ib->index_seq()] = index_count_summary(index_count_map.size()+1,// TODO: get correspondance with plot - ib->index1(), - ib->index2(), - ib->sample_id(), - ib->sample_proj(), - ib->count()); - } - else - { - found_index->second += ib->count(); - } - total_mapped_reads += ib->count(); - } - } - catch(const model::index_out_of_bounds_exception&){continue;} // TODO: check better - } - - float max_fraction_mapped = -std::numeric_limits::max(); - float min_fraction_mapped = std::numeric_limits::max(); - summary.reserve(index_count_map.size()); - - - std::vector keys; - keys.reserve(index_count_map.size()); - for(map_iterator b = index_count_map.begin(), e = index_count_map.end();b != e;++b) - keys.push_back(b->first); - std::stable_sort(keys.begin(), keys.end()); - - for(typename std::vector::const_iterator kcurr = keys.begin(), kbeg=kcurr;kcurr != keys.end();++kcurr) - { - index_count_summary& count_summary = index_count_map[*kcurr]; - count_summary.id(static_cast(std::distance(kbeg, kcurr)+1)); - count_summary.update_fraction_mapped(static_cast(pf_cluster_count_total)); - const float fraction_mapped = count_summary.fraction_mapped(); - summary.push_back(count_summary); - max_fraction_mapped = std::max(max_fraction_mapped, fraction_mapped); - min_fraction_mapped = std::min(min_fraction_mapped, fraction_mapped); - } - - const float avg_fraction_mapped =util::mean(summary.begin(), - summary.end(), - util::op::const_member_function(&index_count_summary::fraction_mapped)); - const float std_fraction_mapped = - std::sqrt(util::variance_with_mean(summary.begin(), - summary.end(), - avg_fraction_mapped, - util::op::const_member_function(&index_count_summary::fraction_mapped))); - summary.set(total_mapped_reads, - pf_cluster_count_total, - cluster_count_total, - min_fraction_mapped, - max_fraction_mapped, - std_fraction_mapped/avg_fraction_mapped); - } /** Summarize a collection index metrics for a specific lane * - * @param index_metrics source collection of index metrics - * @param tile_metrics source collection of tile metrics + * @param metrics source run metrics * @param lane lane number * @param summary destination index lane summary */ - inline void summarize_index_metrics(const model::metrics::run_metrics &metrics, + void summarize_index_metrics(const model::metrics::run_metrics &metrics, const size_t lane, model::summary::index_lane_summary &summary) - throw(model::index_out_of_bounds_exception) - { - summarize_index_metrics(metrics.get_set().begin(), - metrics.get_set().end(), - metrics.get_set(), lane, - summary); - } + throw(model::index_out_of_bounds_exception); /** Summarize a collection index metrics * * @ingroup summary_logic @@ -133,18 +32,11 @@ namespace illumina { namespace interop { namespace logic { namespace summary { * @param lane_count number of lanes * @param summary destination index flowcell summary */ - inline void summarize_index_metrics(const model::metric_base::metric_set& index_metrics, + void summarize_index_metrics(const model::metric_base::metric_set& index_metrics, const model::metric_base::metric_set& tile_metrics, const size_t lane_count, model::summary::index_flowcell_summary &summary) - throw(model::index_out_of_bounds_exception) - { - summary.resize(lane_count); - for(size_t lane=1;lane <= lane_count;++lane) - { - summarize_index_metrics(index_metrics.begin(), index_metrics.end(), tile_metrics, lane, summary[lane-1]); - } - } + throw(model::index_out_of_bounds_exception); /** Summarize index metrics from run metrics * @@ -152,16 +44,9 @@ namespace illumina { namespace interop { namespace logic { namespace summary { * @param metrics source collection of all metrics * @param summary destination index flowcell summary */ - inline void summarize_index_metrics(const model::metrics::run_metrics &metrics, + void summarize_index_metrics(const model::metrics::run_metrics &metrics, model::summary::index_flowcell_summary &summary) - throw(model::index_out_of_bounds_exception) - { - const size_t lane_count = metrics.run_info().flowcell().lane_count(); - summarize_index_metrics(metrics.get_set(), - metrics.get_set(), - lane_count, - summary); - } + throw(model::index_out_of_bounds_exception); }}}} diff --git a/interop/logic/summary/map_cycle_to_read.h b/interop/logic/summary/map_cycle_to_read.h index 66baf7fb0..9753a2de1 100644 --- a/interop/logic/summary/map_cycle_to_read.h +++ b/interop/logic/summary/map_cycle_to_read.h @@ -14,8 +14,6 @@ namespace illumina { namespace interop { namespace logic { namespace summary { - - /** Read number and cycle in read */ struct read_cycle diff --git a/interop/logic/summary/quality_summary.h b/interop/logic/summary/quality_summary.h index 487078231..e79627b5a 100644 --- a/interop/logic/summary/quality_summary.h +++ b/interop/logic/summary/quality_summary.h @@ -15,8 +15,8 @@ #include "interop/model/metrics/q_metric.h" #include "interop/model/summary/run_summary.h" -namespace illumina { namespace interop { namespace logic { namespace summary { - +namespace illumina { namespace interop { namespace logic { namespace summary +{ /** Contains number above Q30 and total number of calls */ struct qval_total @@ -67,7 +67,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary { I end, const read_cycle_vector_t& cycle_to_read, model::summary::run_summary &run) - throw( model::index_out_of_bounds_exception ) + throw( model::index_out_of_bounds_exception ) { typedef model::summary::lane_summary lane_summary; typedef std::vector< size_t > size_vector_t; @@ -181,7 +181,4 @@ namespace illumina { namespace interop { namespace logic { namespace summary { run.total_summary().yield_g(yield_g); run.total_summary().percent_gt_q30(100 * divide(float(useable_calls_gt_q30), float(total_useable_calls))); } -} -} -} -} +}}}} diff --git a/interop/logic/summary/run_summary.h b/interop/logic/summary/run_summary.h index 852c5313a..12f9ec91e 100644 --- a/interop/logic/summary/run_summary.h +++ b/interop/logic/summary/run_summary.h @@ -6,67 +6,13 @@ * @copyright GNU Public License. */ #pragma once -#include #include "interop/model/model_exceptions.h" -#include "interop/logic/summary/error_summary.h" -#include "interop/logic/summary/tile_summary.h" -#include "interop/logic/summary/extraction_summary.h" -#include "interop/logic/summary/quality_summary.h" #include "interop/model/summary/run_summary.h" #include "interop/model/run_metrics.h" -#include "interop/logic/utils/channel.h" -namespace illumina +namespace illumina { namespace interop { namespace logic { namespace summary { -namespace interop -{ -namespace logic -{ -namespace summary -{ - /** Determine maximum number of tiles among all metrics for each lane - * - * @param metrics run metrics - * @param summary run summary - */ - inline void summarize_tile_count(const model::metrics::run_metrics& metrics, model::summary::run_summary& summary) - { - using namespace model::metrics; - for(unsigned int lane=0;lane().tile_numbers_for_lane(lane+1).size()); - tile_count = std::max(tile_count, metrics.get_set().tile_numbers_for_lane(lane+1).size()); - tile_count = std::max(tile_count, metrics.get_set().tile_numbers_for_lane(lane+1).size()); - tile_count = std::max(tile_count, metrics.get_set().tile_numbers_for_lane(lane+1).size()); - for(size_t read=0;read 0; - } - /** Predicate to sort lane summaries by lane number - * - * @param lhs left hand side summary - * @param rhs right hand side summary - * @return true if lhs < rhs - */ - inline bool less_than(const model::summary::lane_summary& lhs, const model::summary::lane_summary& rhs) - { - return lhs.lane() < rhs.lane(); - } - } /** Summarize a collection run metrics * @@ -76,77 +22,7 @@ namespace summary * @param metrics source collection of all metrics * @param summary destination run summary */ - inline void summarize_run_metrics(model::metrics::run_metrics& metrics, model::summary::run_summary& summary) - throw( model::index_out_of_bounds_exception, - model::invalid_channel_exception ) - { - using namespace model::metrics; - if(metrics.empty()) return; - - - summary.initialize(metrics.run_info().reads(), metrics.run_info().flowcell().lane_count()); - - - read_cycle_vector_t cycle_to_read; - map_read_to_cycle_number(summary.begin(), summary.end(), cycle_to_read); - summarize_tile_metrics(metrics.get_set().begin(), metrics.get_set().end(), summary); - summarize_error_metrics(metrics.get_set().begin(), - metrics.get_set().end(), - cycle_to_read, - summary); - INTEROP_ASSERT(metrics.run_info().channels().size()>0); - const size_t intensity_channel = utils::expected2actual_map(metrics.run_info().channels())[0]; - summarize_extraction_metrics(metrics.get_set().begin(), - metrics.get_set().end(), - cycle_to_read, - intensity_channel, - summary); - - if(0 == metrics.get_set().size()) - logic::metric::create_collapse_q_metrics(metrics.get_set(), - metrics.get_set()); - summarize_collapsed_quality_metrics(metrics.get_set().begin(), - metrics.get_set().end(), - cycle_to_read, - summary); - summarize_tile_count(metrics, summary); - - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), - cycle_to_read, - &model::summary::cycle_state_summary::error_cycle_range, - summary); - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), - cycle_to_read, - &model::summary::cycle_state_summary::extracted_cycle_range, - summary); - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), - cycle_to_read, - &model::summary::cycle_state_summary::qscored_cycle_range, - summary); - // Summarize called cycle state - summarize_cycle_state(metrics.get_set(), - metrics.get_set(), - cycle_to_read, - &model::summary::cycle_state_summary::called_cycle_range, - summary); - - // Remove the empty lane summary entries - size_t max_lane_count = 0; - for(size_t read=0;read(beg, end, op)); - INTEROP_ASSERT(!std::isnan(stat.mean())); stat.stddev(std::sqrt(util::variance_with_mean(beg, end, stat.mean(), op))); stat.median(util::median_interpolated(beg, end, comp, op)); } diff --git a/interop/logic/summary/tile_summary.h b/interop/logic/summary/tile_summary.h index 6b57727ab..b393f1dd7 100644 --- a/interop/logic/summary/tile_summary.h +++ b/interop/logic/summary/tile_summary.h @@ -38,7 +38,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary */ template void summarize_tile_metrics(I beg, I end, model::summary::run_summary &run) - throw(model::index_out_of_bounds_exception) + throw(model::index_out_of_bounds_exception) { typedef typename model::metrics::tile_metric::read_metric_vector read_metric_vector_t; typedef typename read_metric_vector_t::const_iterator const_read_metric_iterator; @@ -74,54 +74,60 @@ namespace illumina { namespace interop { namespace logic { namespace summary //reads and reads pf // percent pf INTEROP_ASSERT(run.size() > 0); + model::summary::metric_stat stat; for (size_t lane = 0; lane < run[0].size(); ++lane) { INTEROP_ASSERT(lane < tile_data_by_lane.size()); INTEROP_ASSERT(lane < run[0].size()); summarize(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), - run[0][lane].density(), - util::op::const_member_function(&model::metrics::tile_metric::clusterDensity), - util::op::const_member_function_less(&model::metrics::tile_metric::clusterDensity)); + stat, + util::op::const_member_function(&model::metrics::tile_metric::cluster_density), + util::op::const_member_function_less(&model::metrics::tile_metric::cluster_density)); + run[0][lane].density(stat); summarize(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), - run[0][lane].density_pf(), - util::op::const_member_function(&model::metrics::tile_metric::clusterDensityPf), - util::op::const_member_function_less(&model::metrics::tile_metric::clusterDensityPf)); + stat, + util::op::const_member_function(&model::metrics::tile_metric::cluster_density_pf), + util::op::const_member_function_less(&model::metrics::tile_metric::cluster_density_pf)); + run[0][lane].density_pf(stat); summarize(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), - run[0][lane].cluster_count(), - util::op::const_member_function(&model::metrics::tile_metric::clusterCount), - util::op::const_member_function_less(&model::metrics::tile_metric::clusterCount)); + stat, + util::op::const_member_function(&model::metrics::tile_metric::cluster_count), + util::op::const_member_function_less(&model::metrics::tile_metric::cluster_count)); + run[0][lane].cluster_count(stat); summarize(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), - run[0][lane].cluster_count_pf(), - util::op::const_member_function(&model::metrics::tile_metric::clusterCountPf), - util::op::const_member_function_less(&model::metrics::tile_metric::clusterCountPf)); + stat, + util::op::const_member_function(&model::metrics::tile_metric::cluster_count_pf), + util::op::const_member_function_less(&model::metrics::tile_metric::cluster_count_pf)); + run[0][lane].cluster_count_pf(stat); summarize(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), - run[0][lane].percent_pf(), + stat, util::op::const_member_function(&model::metrics::tile_metric::percent_pf), util::op::const_member_function_less(&model::metrics::tile_metric::percent_pf)); + run[0][lane].percent_pf(stat); run[0][lane].reads(std::accumulate(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), float(0), util::op::const_member_function( - &model::metrics::tile_metric::clusterCount))); + &model::metrics::tile_metric::cluster_count))); run[0][lane].reads_pf(std::accumulate(tile_data_by_lane[lane].begin(), tile_data_by_lane[lane].end(), float(0), util::op::const_member_function( - &model::metrics::tile_metric::clusterCountPf))); + &model::metrics::tile_metric::cluster_count_pf))); for (size_t read = 1; read < run.size(); ++read) { INTEROP_ASSERT(read < run.size()); - run[read][lane].density() = run[0][lane].density(); - run[read][lane].density_pf() = run[0][lane].density_pf(); - run[read][lane].cluster_count() = run[0][lane].cluster_count(); - run[read][lane].cluster_count_pf() = run[0][lane].cluster_count_pf(); - run[read][lane].percent_pf() = run[0][lane].percent_pf(); + run[read][lane].density(run[0][lane].density()); + run[read][lane].density_pf(run[0][lane].density_pf()); + run[read][lane].cluster_count(run[0][lane].cluster_count()); + run[read][lane].cluster_count_pf(run[0][lane].cluster_count_pf()); + run[read][lane].percent_pf(run[0][lane].percent_pf()); run[read][lane].reads(run[0][lane].reads()); run[read][lane].reads_pf(run[0][lane].reads_pf()); } @@ -138,23 +144,27 @@ namespace illumina { namespace interop { namespace logic { namespace summary for (size_t lane = 0; lane < run[read].size(); ++lane) { INTEROP_ASSERT(lane < run[0].size()); + const size_t non_nan = nan_summarize(read_data_by_lane_read(read, lane).begin(), read_data_by_lane_read(read, lane).end(), - run[read][lane].percent_aligned(), + stat, util::op::const_member_function( &model::metrics::read_metric::percent_aligned), util::op::const_member_function_less( &model::metrics::read_metric::percent_aligned)); + run[read][lane].percent_aligned(stat); nan_summarize(read_data_by_lane_read(read, lane).begin(), read_data_by_lane_read(read, lane).end(), - run[read][lane].prephasing(), + stat, util::op::const_member_function(&model::metrics::read_metric::percent_prephasing), util::op::const_member_function_less(&model::metrics::read_metric::percent_prephasing)); + run[read][lane].prephasing(stat); nan_summarize(read_data_by_lane_read(read, lane).begin(), read_data_by_lane_read(read, lane).end(), - run[read][lane].phasing(), + stat, util::op::const_member_function(&model::metrics::read_metric::percent_phasing), util::op::const_member_function_less(&model::metrics::read_metric::percent_phasing)); + run[read][lane].phasing(stat); INTEROP_ASSERT(!std::isnan(run[read][lane].percent_aligned().mean())); percent_aligned_by_read += run[read][lane].percent_aligned().mean() * non_nan; total_by_read += non_nan; diff --git a/interop/logic/table/check_imaging_table_column.h b/interop/logic/table/check_imaging_table_column.h new file mode 100644 index 000000000..2a95eee86 --- /dev/null +++ b/interop/logic/table/check_imaging_table_column.h @@ -0,0 +1,198 @@ +/** Logic to check if a column should be populated + * + * @file + * @date 7/22/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include +#include "interop/util/exception.h" +#include "interop/util/constant_mapping.h" +#include "interop/util/string.h" +#include "interop/constants/enums.h" +#include "interop/logic/utils/enums.h" +#include "interop/model/run_metrics.h" +#include "interop/model/model_exceptions.h" +#include "interop/model/table/imaging_column.h" +#include "interop/logic/table/table_util.h" + +namespace illumina { namespace interop { namespace logic { namespace table +{ + /** Static class to hide methods created through macro expansion + */ + class check_imaging_table_column + { + typedef model::metrics::q_metric::uint_t uint_t; + enum VoidType + { + Void + }; + public: + /** Set all the id columns as filled + * + * @note Unlike the case of base_metric, nothing derives from metric_set, so this must be explicit + * @param filled boolean vector of flags indicating when the column is filled with data + */ + static void set_id_filled(std::vector &filled) + { + set_filled_for_metric_set(model::metric_base::metric_set(), filled, true); + set_filled_for_metric_set(model::metric_base::metric_set(), filled, + true); + set_filled_for_metric_set(model::metric_base::metric_set(), filled, + true); + } + + /** Update the boolean filled metric array for each column based on whether the metric set is not empty + * + * @param metrics source metric set to populate the row of data + * @param filled boolean vector of flags indicating when the column is filled with data + * @param force_true force the filled column to be set true + */ + template + static void + set_filled_for_metric_set(const MetricSet &metrics, std::vector &filled, const bool force_true = false) + { + /* For every entry in INTEROP_IMAGING_COLUMN_TYPES + * Add a method call to set a boolean flag to true if the column is occupied by values + * + * Example: + * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType,0) -> + * + * set_filled_laneVoid(metrics, filled, force); + */ +# define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) set_filled_##Method##Param(metrics, filled, force_true); + INTEROP_IMAGING_COLUMN_TYPES +# undef INTEROP_TUPLE7 // Reuse this for another conversion + } + + /** Update the filled metric array for each column based on whether the value is valid + * + * @note While this will work for any metric, it is intended only for tile_metrics. Unlike other metrics, + * tile_metrics may have partial entries. + * + * @param metric source metric to populate the row of data + * @param read read number + * @param q20_idx index of the q20 value + * @param q30_idx index of the q30 value + * @param naming_method tile naming method + * @param filled boolean vector of flags indicating when the column is filled with data + */ + template + static void set_filled_for_metric(const Metric &metric, + const size_t read, + const size_t q20_idx, + const size_t q30_idx, + const constants::tile_naming_method naming_method, + std::vector &filled) + { + /* For every entry in INTEROP_IMAGING_COLUMN_TYPES + * Add a method call to set a boolean flag to true if the column is occupied by valid values + * + * Example: + * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType, 0) -> + * + * set_filled_metric_laneVoid(metric, read, q20_idx, q30_idx, naming_method, filled); + */ +# define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) \ + set_filled_metric_##Method##Param(metric, read, static_cast(q20_idx), static_cast(q30_idx), naming_method, filled); + INTEROP_IMAGING_COLUMN_TYPES +# undef INTEROP_TUPLE7 // Reuse this for another conversion + } + + private: + /* For every entry in INTEROP_IMAGING_COLUMN_TYPES + * This macro creates two functions, one to set the filled array to true if the metric set is not empty + * and the other does nothing. + * + * Example: + * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType, 0) -> + * + * static void set_filled_laneVoid(const metric_base::metric_set& metrics, std::vector& filled, const bool force_true) + * static void set_filled_laneVoid(const model::metric_base::empty_header&, std::vector&, const bool) + * + */ +# define INTEROP_TUPLE7(Id, Metric, Method, Param, Ignored1, Ignored2, Ignored3) \ + static void set_filled_##Method##Param(const model::metric_base::metric_set& metrics, std::vector& filled, const bool force_true)\ + {\ + filled[model::table::Id##Column] = (force_true) ? true : metrics.size()>0;\ + }\ + static void set_filled_##Method##Param(const model::metric_base::empty_header&, std::vector&, const bool){} + INTEROP_IMAGING_COLUMN_TYPES +# undef INTEROP_TUPLE7 // Reuse this for another conversion + + private: + /* For every entry in INTEROP_IMAGING_COLUMN_TYPES + * This macro creates two functions, one to mark a column as filled if there is at least one valid metric + * + * Example: + * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType, 0) -> + * + * static void set_filled_metric_laneVoid(const Metric& metric, + * const size_t Read, + * const uint_t Q20, + * const uint_t Q30, + * const constants::tile_naming_method NamingConvention, + * std::vector& filled) + * static void set_filled_metric_laneVoid(const model::metric_base::empty_metric& + * const size_t, + * const uint_t, + * const uint_t, + * const constants::tile_naming_method, + * std::vector&) + */ +# define INTEROP_TUPLE7(Id, Metric, Method, Param, Ignored1, Ignored2, Ignored3) \ + static void set_filled_metric_##Method##Param(const model::Metric& metric,\ + const size_t Read,\ + const uint_t Q20,\ + const uint_t Q30,\ + const constants::tile_naming_method NamingConvention,\ + std::vector& filled)\ + {\ + if(is_valid(call_adapter(metric, Param, &model:: Metric::Method))) filled[model::table::Id##Column] = true;\ + (void)Q20;(void)Q30;(void)NamingConvention;(void)Read;\ + }\ + static void set_filled_metric_##Method##Param(const model::metric_base::empty_metric&,\ + const size_t,\ + const uint_t,\ + const uint_t,\ + const constants::tile_naming_method,\ + std::vector&){} + INTEROP_IMAGING_COLUMN_TYPES +# undef INTEROP_TUPLE7 // Reuse this for another conversion + + private: + /** Test if a metric type is valid + * + * @param val floating point value + * @return true if not NaN + */ + static bool is_valid(const float val) + { + return !std::isnan(val); + } + + /** Test if a metric type is valid + * + * @param val integral value + * @return true + */ + template + static bool is_valid(const T) + { + return true; + } + + /** Test if a metric type is valid + * + * @param values vector of values + * @return true if not empty + */ + template + static bool is_valid(const std::vector &values) + { + return !values.empty(); + } + }; +}}}} diff --git a/interop/logic/table/create_imaging_table.h b/interop/logic/table/create_imaging_table.h new file mode 100644 index 000000000..647f9cbc4 --- /dev/null +++ b/interop/logic/table/create_imaging_table.h @@ -0,0 +1,49 @@ +/** Logic to populate the imaging table + * + * @file + * @date 7/20/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include "interop/model/run_metrics.h" +#include "interop/model/table/imaging_column.h" +#include "interop/model/table/imaging_table.h" + +namespace illumina { namespace interop { namespace logic { namespace table +{ + /** Populate the imaging table with all the metrics in the run + * + * @param metrics collection of all run metrics + * @param columns vector of table columns + * @param row_offset ordering for the rows + * @param data_beg iterator to start of table data + * @param n number of cells in the data table + */ + void populate_imaging_table_data(const model::metrics::run_metrics& metrics, + const std::vector& columns, + const std::map& row_offset, + float* data_beg, const size_t n); + /** Count the number of rows in the imaging table and setup an ordering + * + * @param metrics collections of InterOp metric sets + * @param row_offset ordering for the rows + */ + void count_table_rows(const model::metrics::run_metrics& metrics, + std::map& row_offset); + /** Count the total number of columns for the data table + * + * @param columns vector of table column descriptions + * @return total number of columns including sub columns + */ + size_t count_table_columns(const std::vector& columns); + /** Create an imaging table from run metrics + * + * @param metrics source run metrics + * @param table destination imaging table + */ + void create_imaging_table(const model::metrics::run_metrics& metrics, model::table::imaging_table& table) + throw(model::invalid_column_type, model::index_out_of_bounds_exception); + +}}}} diff --git a/interop/logic/table/create_imaging_table_columns.h b/interop/logic/table/create_imaging_table_columns.h new file mode 100644 index 000000000..ce54f34cc --- /dev/null +++ b/interop/logic/table/create_imaging_table_columns.h @@ -0,0 +1,49 @@ +/** Logic to populate the imaging table columns + * + * @file + * @date 7/20/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include "interop/model/run_metrics.h" +#include "interop/model/table/imaging_column.h" +#include "interop/logic/table/table_util.h" + + +namespace illumina { namespace interop { namespace logic { namespace table +{ + + /** Get the number of digits to round a column + * + * @param index imaging table column id + * @return number of digits to round a column + */ + size_t get_column_rounding(const model::table::column_id index); + /** Create the imaging table columns + * + * @param channels names of each channel + * @param filled boolean array indicating whether to include the column + * @param columns destination column vector + */ + void create_imaging_table_columns(const std::vector& channels, + const std::vector& filled, + std::vector< model::table::imaging_column >& columns) + throw(model::invalid_column_type, model::index_out_of_bounds_exception); + /** Populate the value offsets from the column headers + * + * @param columns column headers + */ + void populate_column_offsets(std::vector& columns) + throw(model::invalid_column_type); + /** Create a vector of column descriptors + * + * @param metrics source collection of InterOp metrics from the run + * @param columns destination vector of column descriptors + */ + void create_imaging_table_columns(const model::metrics::run_metrics& metrics, + std::vector< model::table::imaging_column >& columns) + throw(model::invalid_column_type, model::index_out_of_bounds_exception); + + +}}}} diff --git a/interop/logic/table/populate_imaging_table.h b/interop/logic/table/populate_imaging_table.h deleted file mode 100644 index 5fb000b5f..000000000 --- a/interop/logic/table/populate_imaging_table.h +++ /dev/null @@ -1,425 +0,0 @@ -/** Logic to populate the imaging table - * - * @file - * @date 5/12/16 - * @version 1.0 - * @copyright GNU Public License. - */ -#pragma once -#include -#include -#include "interop/util/exception.h" -#include "interop/util/constant_mapping.h" -#include "interop/util/string.h" -#include "interop/util/length_of.h" -#include "interop/constants/enums.h" -#include "interop/logic/utils/enums.h" -#include "interop/model/metrics/error_metric.h" -#include "interop/model/metrics/extraction_metric.h" -#include "interop/model/metrics/tile_metric.h" -#include "interop/model/run_metrics.h" -#include "interop/model/model_exceptions.h" -#include "interop/model/table/column_header.h" -#include "interop/model/table/imaging_table_entry.h" -#include "interop/model/table/imaging_table.h" -#include "interop/logic/summary/map_cycle_to_read.h" -#include "interop/io/table/csv_format.h" - -namespace illumina { namespace interop { namespace logic { namespace table { - - - /** Convert a column type to the data type - * - * @param type imaging table column type - * @return metric data - */ - inline constants::metric_data to_data_type(const model::table::column_type type) - { - typedef std::pair mapped_t; -# define INTEROP_TUPLE7(Name, Ignored2, Ignored3, Ignored4, Ignored5, Data, Ignored6) mapped_t(model::table::Name##Column, constants::Data), - static const mapped_t name_types[] = {INTEROP_IMAGING_COLUMN_TYPES};// mapped_t(model::table::ImagingColumnCount, constants::MetricDataCount)}; -# undef INTEROP_TUPLE7 - return util::constant_mapping_get(name_types, type, constants::UnknownMetricData); - } - /** Convert a column index to the data type - * - * @param index imaging table column index - * @return metric data - */ - inline constants::metric_data to_data_type(const size_t index) - { - return to_data_type(static_cast(index)); - } - - /** Defines a table_entry map, a temporary data structure used in logic */ - typedef std::map< model::metric_base::base_metric::id_t, model::table::table_entry > table_entry_map_t; - /** Define a map from the current cycle to the read information */ - typedef summary::read_cycle_vector_t read_cycle_vector_t; - /** Define an offset vector for filling a table */ - typedef model::table::table_entry::table_fill_vector_t table_fill_vector_t; - - /** Get the internal names of each column - * - * @return collection of strings representing a column identifier - */ - inline const std::vector& imaging_table_column_names() - { - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * - * Convert each entry to a string name - * - * Example: - * INTEROP_TUPLE6(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * "Lane" - * - * @see INTEROP_IMAGING_COLUMN_TYPES - */ -# define INTEROP_TUPLE7(Id, Ignored1, Ignored2, Ignored3, Ignored4, Ignored5, Ignored6) #Id, - // The list of strings below ends with a ',', e.g. "Lane", "Tile", "Cycle", - // This is not allowed in C++, so I added a dummy string to the end. - static const std::string name_types[] = {INTEROP_IMAGING_COLUMN_TYPES "Dummy"};// Added dummy as a work around -# undef INTEROP_TUPLE7 - - static const std::vector tmp(name_types, name_types+util::length_of(name_types)-1);// remove dummy - return tmp; - } - /** Get the internal names of each column - * - * @todo keep or remove? - * - * @param filled boolean array indicating filled columns - * @return collection of strings representing a column identifier - */ - inline std::vector imaging_table_column_names(const std::vector& filled) - { - std::vector tmp = imaging_table_column_names(); - size_t k=0; - for(size_t i=0;i="); - util::replace(header, "KPermm2", "(k/mm2)"); - util::replace(header, "K", " (k)"); - util::camel_to(header); - return header; - } - /** Convert header to name - * - * @param header column header - * @return column name - */ - inline std::string to_name(const model::table::column_header& header) - { - std::string name = header.id(); - util::replace(name, "%", "Percent"); - util::replace(name, ">=", "GreaterThan"); - util::replace(name, "(k/mm2)", "KPermm2"); - util::replace(name, " (k)", "K"); - util::camel_from(name); - return name; - } - /** Populate the value offsets from the column headers - * - * @param offset destination offset array - * @param headers column headers - */ - inline void populate_column_offsets(table_fill_vector_t& offsets, - const std::vector& headers) - throw(model::invalid_column_type) - { - const std::vector& names(imaging_table_column_names()); - offsets.assign(model::table::ImagingColumnCount, std::vector()); - for(size_t i=0;i::const_iterator it = std::find(names.begin(), names.end(), to_name(headers[i].id())); - if(it == names.end()) - INTEROP_THROW(model::invalid_column_type, "Column name not found: " << headers[i].id() << " - " << to_name(headers[i].id())); - const size_t metric_column = static_cast(std::distance(names.begin(), it)); - offsets[metric_column].push_back(i); - } - } - /** Populate a vector with the table column headers - * - * @param headers destination vector - * @param channels channel names to use in header - * @param filled boolean vector indicating which columns to keep - */ - inline void populate_column_headers(std::vector& headers, - const std::vector& channels, - const std::vector& filled) - throw(model::invalid_column_type) - { - const std::vector& names(imaging_table_column_names()); - headers.clear(); - headers.reserve(names.size()+channels.size()+constants::NUM_OF_BASES); - if(filled.size() != names.size()) - INTEROP_THROW(model::invalid_column_type, "Filled vector does not match number of column names"); - for(size_t i=0;i(base)))); - - break; - } - case constants::ChannelArray: - { - for(size_t channel = 0;channel < channels.size();++channel) - headers.push_back(model::table::column_header(title, channels[channel])); - break; - } - default: - { - INTEROP_THROW( model::invalid_column_type, "Column index does not have a type"); - } - } - } - } - /** Populate a vector with the table column headers - * - * @param headers destination vector - * @param channels channel names to use in header - */ - inline void populate_column_headers(std::vector& headers, - const std::vector& channels) - throw(model::invalid_column_type) - { - const std::vector filled(imaging_table_column_names().size(), true); - populate_column_headers(headers, channels, filled); - } - - /** Populate the table using the iterator over a metric set - * - * @note this only support cycle metrics - * - * @param beg iterator to start of metric collection - * @param end iterator to end of metric collection - * @param q20_idx index of Q20 bin - * @param q30_idx index of Q30 bin - * @param naming_convention tile naming method - * @param cycle_to_read map that takes a cycle and returns the read-number cycle-in-read pair - * @param table table map a metric ID to a row in the table - */ - template - void populate_imaging_table_by_cycle(I beg, - I end, - const size_t q20_idx, - const size_t q30_idx, - const constants::tile_naming_method naming_method, - const read_cycle_vector_t& cycle_to_read, - table_entry_map_t& table) throw(model::index_out_of_bounds_exception) - { - for(I cur=beg;cur != end;++cur) - { - table_entry_map_t::iterator it = table.find(cur->id()); - if(it == table.end()) - { - if((cur->cycle()-1) >= cycle_to_read.size()) - throw model::index_out_of_bounds_exception("Cycle exceeds total cycles from Reads in the RunInfo.xml"); - const summary::read_cycle& read = cycle_to_read[cur->cycle()-1]; - table[cur->id()] = model::table::table_entry(*cur, - read.number, - read.cycle_within_read, - q20_idx, - q30_idx, - naming_method); - } // TODO populate tile here? - else it->second.update(*cur); - } - } - - /** Populate the imaging table from all the metrics in run_metrics - * - * @ingroup table_logic - * @param metrics run metrics - * @param table collection of table rows - * @param filled_columns collection boolean values indicating which columns are filled - */ - template - inline void populate_imaging_table(const model::metrics::run_metrics& metrics, - Table& table, - std::vector< bool >& filled_columns) throw(model::index_out_of_bounds_exception) - { - table_entry_map_t entry_map; - const constants::tile_naming_method naming_method = metrics.run_info().flowcell().naming_method(); - const size_t q20_idx = metric::index_for_q_value(metrics.get_set(), 20); - const size_t q30_idx = metric::index_for_q_value(metrics.get_set(), 30); - filled_columns.assign(model::table::ImagingColumnCount, false); - - read_cycle_vector_t cycle_to_read; - summary::map_read_to_cycle_number(metrics.run_info().reads().begin(), - metrics.run_info().reads().end(), - cycle_to_read); - // TODO replace with visitor that only selects cycle metrics - populate_imaging_table_by_cycle(metrics.get_set().begin(), - metrics.get_set().end(), - q20_idx, - q30_idx, - naming_method, - cycle_to_read, - entry_map); - model::table::table_entry::set_filled(metrics.get_set(), filled_columns); - populate_imaging_table_by_cycle(metrics.get_set().begin(), - metrics.get_set().end(), - q20_idx, - q30_idx, - naming_method, - cycle_to_read, - entry_map); - model::table::table_entry::set_filled(metrics.get_set(), filled_columns); - populate_imaging_table_by_cycle(metrics.get_set().begin(), - metrics.get_set().end(), - q20_idx, - q30_idx, - naming_method, - cycle_to_read, - entry_map); - model::table::table_entry::set_filled(metrics.get_set(), filled_columns); - populate_imaging_table_by_cycle(metrics.get_set().begin(), - metrics.get_set().end(), - q20_idx, - q30_idx, - naming_method, - cycle_to_read, - entry_map); - model::table::table_entry::set_filled(metrics.get_set(), filled_columns); - populate_imaging_table_by_cycle(metrics.get_set().begin(), - metrics.get_set().end(), - q20_idx, - q30_idx, - naming_method, - cycle_to_read, - entry_map); - model::table::table_entry::set_filled(metrics.get_set(), - filled_columns); - const model::metric_base::metric_set& tile_metrics = - metrics.get_set(); - table.reserve(entry_map.size()); - for(table_entry_map_t::iterator b = entry_map.begin(), e=entry_map.end();b != e;++b) - { - if(tile_metrics.has_metric(b->second.Lane, b->second.Tile)) - { - b->second.set_filled_metric(tile_metrics.get_metric(b->second.Lane, b->second.Tile), filled_columns); - b->second.update(tile_metrics.get_metric(b->second.Lane, b->second.Tile)); - } - table.push_back(b->second); - } - if(!table.empty()) - { - model::table::table_entry::set_id_filled(filled_columns); - filled_columns[model::table::SectionColumn] = table[0].Section > 0; - } - } - - /** Populate the imaging table from all the metrics in run_metrics - * - * @ingroup table_logic - * @param metrics run metrics - * @param table imaging table - */ - inline void populate_imaging_table(const model::metrics::run_metrics& metrics, - model::table::imaging_table& table) - throw(model::index_out_of_bounds_exception) - { - std::vector< bool > filled_columns; - populate_imaging_table(metrics, table, filled_columns); - table.filled_columns(filled_columns); - } - /** Populate a vector with the table column headers - * - * @param channels channel names to use in header - * @param table imaging table - */ - inline void populate_column_headers(const std::vector& channels, - model::table::imaging_table& table) - throw(model::invalid_column_type) - { - std::vector headers; - populate_column_headers(headers, channels, table.filled_columns()); - table.headers(headers); - } - -}}}} - - -namespace illumina { namespace interop { namespace model { namespace table { - - /** Read an imaging table from an input stream in the CSV format - * - * @param in input stream - * @param table imaging table - * @return input stream - */ - std::istream &operator>>(std::istream &in, model::table::imaging_table &table) - { - io::table::read_csv_line(in, table.m_columns_header); - if (!in.good()) return in; - logic::table::table_fill_vector_t offsets; - logic::table::populate_column_offsets(offsets, table.m_columns_header); - - std::vector values; - values.reserve(table.m_columns_header.size()); - model::table::table_entry row; - while (in) - { - io::table::read_csv_line(in, values); - if (values.empty()) continue; - row.copy_from_vector(values, offsets); - table.push_back(row); - } - return in; - } - /** Write the imaging table to the output stream in the CSV format - * - * @param out output stream - * @param table imaging table - * @return output stream - */ - std::ostream &operator<<(std::ostream &out, const model::table::imaging_table &table) - { - std::vector values; - values.reserve(table.m_columns_header.size()); - if (!out.good()) return out; - io::table::write_csv_line(out, table.m_columns_header); - if (!out.good()) return out; - for (model::table::imaging_table::const_iterator it = table.begin(); - it != table.end(); ++it) - { - it->copy_to_vector(values, table.filled_columns()); - io::table::write_csv_line(out, values); - if (!out.good())return out; - } - return out; - } -}}}} diff --git a/interop/logic/table/table_populator.h b/interop/logic/table/table_populator.h new file mode 100644 index 000000000..404880d3e --- /dev/null +++ b/interop/logic/table/table_populator.h @@ -0,0 +1,248 @@ +/** Logic to populate the imaging table + * + * @file + * @date 7/22/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include +#include "interop/util/exception.h" +#include "interop/util/constant_mapping.h" +#include "interop/util/string.h" +#include "interop/util/length_of.h" +#include "interop/constants/enums.h" +#include "interop/model/run_metrics.h" +#include "interop/model/model_exceptions.h" +#include "interop/model/table/imaging_column.h" +#include "interop/model/table/imaging_table.h" +#include "interop/logic/utils/enums.h" +#include "interop/logic/table/table_util.h" +#include "interop/logic/summary/map_cycle_to_read.h" +#include "interop/logic/table/create_imaging_table_columns.h" + +namespace illumina { namespace interop { namespace logic { namespace table +{ + /** Populate a table with data from a single InterOp + */ + class table_populator + { + enum VoidType + { + Void + }; + typedef model::metrics::q_metric::uint_t uint_t; + public: + /** Populate the ID columns of the table + * + * @param metric metric with source data + * @param read read number, cycle within read + * @param q20_idx index of the q20 value + * @param q30_idx index of the q30 value + * @param naming_method tile naming method enum + * @param columns vector of table columns + * @param data_it iterator to current row of table data + * @param data_end iterator to end of table data + */ + template + static void populate_id(const Metric &metric, + const summary::read_cycle &read, + const size_t q20_idx, + const size_t q30_idx, + const constants::tile_naming_method naming_method, + const std::vector &columns, + OutputIterator data_it, OutputIterator data_end) + { + populate(static_cast(metric), + read.number, + q20_idx, + q30_idx, + naming_method, + columns, + data_it, + data_end); + populate(static_cast(metric), + read.number, + q20_idx, + q30_idx, + naming_method, + columns, + data_it, + data_end); + assign(data_it[columns[model::table::ReadColumn]], read.number); + assign(data_it[columns[model::table::CycleWithinReadColumn]], read.cycle_within_read); + (void) data_end; + } + + /** Populate the data columns of the table + * + * @param metric metric with source data + * @param read read number, cycle within read + * @param q20_idx index of the q20 value + * @param q30_idx index of the q30 value + * @param naming_method tile naming method enum + * @param columns vector of table columns + * @param data_it iterator to current row of table data + * @param data_end iterator to end of table data + */ + template + static void populate(const Metric &metric, + const size_t read, + const size_t q20_idx, + const size_t q30_idx, + const constants::tile_naming_method naming_method, + const std::vector &columns, + OutputIterator data_it, OutputIterator data_end) + { + /* For every entry in INTEROP_IMAGING_COLUMN_TYPES + * Add a method call to update each field with the given `metric` + * + * Example: + * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> + * + * update_laneVoid(metric, q20_idx, q30_idx, naming_convention); + */ +# define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) \ + populate_##Method##Param(metric, read, static_cast(q20_idx), static_cast(q30_idx), naming_method, columns, data_it, data_end); + INTEROP_IMAGING_COLUMN_TYPES +# undef INTEROP_TUPLE7 // Reuse this for another conversion + } + + private: + /* For every entry in INTEROP_IMAGING_COLUMN_TYPES + * This macro creates two functions, one to populate a field/column with the data from the correspond metric + * and an empty method to ignore a group + * + * Example: + * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> + * + * void update_laneVoid(const model::metric_base::base_metric& metric, const size_t Q20, const size_t Q30, const naming_method NamingConvention) + * void update_laneVoid(const model::metric_base::empty_metric&) + * + * @note Param can be can field in this class, e.g. Read, or the function parameters Q20, Q30 or NamingConvention + */ +# define INTEROP_TUPLE7(Id, Metric, Method, Param, Type, Kind, Round) \ + template\ + static void populate_##Method##Param(const model:: Metric& metric,\ + const size_t Read,\ + const uint_t Q20,\ + const uint_t Q30,\ + const constants::tile_naming_method NamingConvention,\ + const std::vector& columns,\ + OutputIterator data_it, OutputIterator data_end)\ + {\ + const size_t index = columns[model::table:: Id##Column];\ + if(index == std::numeric_limits::max()) return; /*Missing column */ \ + copy_to(data_it+index, data_end, call_adapter(metric, Param, &model:: Metric::Method), Round);\ + (void)Q20;(void)Q30;(void)NamingConvention;(void)Read;\ + }\ + template\ + static void populate_##Method##Param(const MetricType&,\ + const size_t,\ + const uint_t,\ + const uint_t,\ + const constants::tile_naming_method,\ + const std::vector&,\ + OutputIterator,OutputIterator){} + INTEROP_IMAGING_COLUMN_TYPES +# undef INTEROP_TUPLE7 // Reuse this for another conversion + + private: + /** Assign a value to an iterator + * + * @param destination reference to iterator value + * @param end iterator to end of destination collection + * @param source value to assign + * @param num_digits number of digits after the decimal + */ + template + static void assign(T &destination, const U source, const size_t num_digits = 0) + { + //INTEROP_ASSERTMSG(std::isnan(destination), destination); + destination = static_cast(roundto(source, num_digits)); + } + + private: + inline static float roundto(const float val, const size_t n) + { + if (n > 0) + { + double mult = 1.0; + for (size_t i = 0; i < n; ++i) mult *= 10; + return static_cast(std::floor(val * mult + 0.5) / mult); + } + return val; + } + + template + inline static T roundto(const T val, const size_t) + { return val; } + + /** Assign a value to an iterator + * + * @param destination iterator to desination collection + * @param end iterator to end of destination collection + * @param source value to assign + * @param num_digits number of digits after the decimal + */ + template + static void copy_to(OutputIterator destination, OutputIterator end, const U source, const size_t num_digits) + { + INTEROP_ASSERT(destination < end); + (void) end; + assign(*destination, source, num_digits); + } + + /** Assign a value to an iterator + * + * @param destination iterator to desination collection + * @param end iterator to end of destination collection + * @param source value to assign + * @param num_digits number of digits after the decimal + */ + template + static void + copy_to(OutputIterator destination, OutputIterator end, const std::vector &source, const size_t num_digits) + { + (void) end; + for (typename std::vector::const_iterator it = source.begin(); it != source.end(); ++it, ++destination) + { + INTEROP_ASSERT(destination < end); + assign(*destination, *it, num_digits); + } + } + + /** Test if a metric type is valid + * + * @param val floating point value + * @return true if not NaN + */ + static bool is_valid(const float val) + { + return !std::isnan(val); + } + + /** Test if a metric type is valid + * + * @param val integral value + * @return true + */ + template + static bool is_valid(const T) + { + return true; + } + + /** Test if a metric type is valid + * + * @param values vector of values + * @return true if not empty + */ + template + static bool is_valid(const std::vector &values) + { + return !values.empty(); + } + }; +}}}} diff --git a/interop/logic/table/table_util.h b/interop/logic/table/table_util.h new file mode 100644 index 000000000..a244256f9 --- /dev/null +++ b/interop/logic/table/table_util.h @@ -0,0 +1,71 @@ +/** Utility functions for building a table + * + * @note These functions were move here as a work around for SWIG + * + * @file + * @date 6/13/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include "interop/model/table/imaging_column.h" + +namespace illumina { namespace interop { namespace logic { namespace table +{ + + /** Function interface for method call with single parameter + * + * @note This is necessary because you cannot pass anything into an function that expects no arguments, not even void + * + * @param obj object corresponding to the method + * @param param1 first value to function + * @param func pointer to member function + * @return functor wrapper + */ + template + R call_adapter(const T& obj, P2 param1, R (T::*func )(P1)const) + { + return (obj.*func)(param1); + } + /** Function interface for method call with single dummy parameter + * + * @note This is necessary because you cannot pass anything into an function that expects no arguments, not even void + * + * @param obj object corresponding to the method + * @param func pointer to member function + * @return functor wrapper + */ + template + R call_adapter(const T& obj, P1, R (T::*func )()const) + { + return (obj.*func)(); + } + + +}}}} + + +/** This macro maps an enum to a string/enum pair */ +#define INTEROP_TUPLE7(X, Ignore1, Ignore2, Ignore3, Ignore4, Ignore5, Ignored6) name_type_pair_t(#X, model::table:: X##Column), +namespace illumina { namespace interop { namespace constants { + + /** Specialization that maps imaging column types to a string */ + template<> + class enumeration_string_mapping + { + typedef std::pair name_type_pair_t; + public: + /** Pass an array of string, enum pairs and its length to the given function + * + * @param func pointer to a function that takes an array of string/enum pairs as a parameter + * @return the value returned by the given function (in the case of `parse` and `to_string` the return value is a singleton) + */ + template + static R setup(F func) + { + static const name_type_pair_t name_types[] = {INTEROP_IMAGING_COLUMN_TYPES name_type_pair_t("Dummy", model::table::ImagingColumnCount)}; + return func(name_types, util::length_of(name_types)); + } + }; +}}} +#undef INTEROP_TUPLE7 // Reuse this for another conversion diff --git a/interop/logic/utils/channel.h b/interop/logic/utils/channel.h index a302fd6e0..f0f7f7ba0 100644 --- a/interop/logic/utils/channel.h +++ b/interop/logic/utils/channel.h @@ -6,6 +6,11 @@ * @copyright GNU Public License. */ #pragma once +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4290)// MSVC warns that it ignores the exception specification. +#endif + #include #include #include @@ -190,3 +195,6 @@ namespace illumina { namespace interop { namespace logic { namespace utils }}}} +#ifdef _MSC_VER +#pragma warning(pop) +#endif diff --git a/interop/logic/utils/enums.h b/interop/logic/utils/enums.h index d63b9f750..d5b651c3e 100644 --- a/interop/logic/utils/enums.h +++ b/interop/logic/utils/enums.h @@ -20,7 +20,8 @@ /** This macro maps an enum description to a string/enum pair */ #define INTEROP_TUPLE4(X, IGNORED_1, IGNORED_2, IGNORED_3) name_type_pair_t(#X,X) -namespace illumina { namespace interop { namespace constants { +namespace illumina { namespace interop { namespace constants +{ /** Template class declaration to map enum types to string representations * * @note only the specializations do something @@ -28,6 +29,11 @@ namespace illumina { namespace interop { namespace constants { template class enumeration_string_mapping; + /** Convert an enum value to its string representation + * + * @param value enum value + * @return string representation of the enum value + */ template std::string to_string(Enum value) { @@ -36,6 +42,11 @@ namespace illumina { namespace interop { namespace constants { return enum_map_t::template setup(mapping_t::rmapping).get(value, std::string("Unknown")); } + /** Convert a string representation of a enum value to an enum value + * + * @param name string representation of a enum value + * @return enum value + */ template Enum parse(const std::string& name) { @@ -52,8 +63,9 @@ namespace illumina { namespace interop { namespace constants { /** Constructor * * @param vec1 vector to fill + * @param start number of start elements to skip */ - fill_enum_vector(std::vector &vec1) : m_vec(vec1) { } + fill_enum_vector(std::vector &vec1, const size_t start=0) : m_vec(vec1), m_start(start) { } /** Fill the vector * * @param vals string/enum pair array @@ -61,23 +73,61 @@ namespace illumina { namespace interop { namespace constants { */ void operator()(const std::pair *vals, const size_t n) const { - m_vec.resize(n); - for (size_t i = 0; i < n; ++i) - m_vec[i] = vals[i].second; + m_vec.resize(n-m_start); + for (size_t i = m_start; i < n; ++i) + m_vec[i-m_start] = vals[i].second; } private: std::vector &m_vec; + const size_t m_start; }; + /** Helper functor to fill an enum vector */ + template + struct fill_enum_name_vector + { + /** Constructor + * + * @param vec1 vector to fill + * @param start number of start elements to skip + */ + fill_enum_name_vector(std::vector &vec1, const size_t start=0) : m_vec(vec1), m_start(start) { } + /** Fill the vector + * + * @param vals string/enum pair array + * @param n number of elements in array + */ + void operator()(const std::pair *vals, const size_t n) const + { + m_vec.resize(n-m_start); + for (size_t i = m_start; i < n; ++i) + m_vec[i-m_start] = vals[i].first; + } + private: + std::vector &m_vec; + const size_t m_start; + }; + } + /** Fill given enum vector with all available enums + * + * @param vec enum vector + * @param skip number of start elements to skip + */ + template + void list_enums(std::vector& vec, const size_t skip=0) + { + typedef enumeration_string_mapping enum_map_t; + enum_map_t::template setup(detail::fill_enum_vector(vec, skip)); } /** Fill given enum vector with all available enums * * @param vec enum vector + * @param skip number of start elements to skip */ template - void list_enums(std::vector& vec) + void list_enum_names(std::vector& vec, const size_t skip=0) { typedef enumeration_string_mapping enum_map_t; - enum_map_t::template setup(detail::fill_enum_vector(vec)); + enum_map_t::template setup(detail::fill_enum_name_vector(vec, skip)); } /** Specialization that maps metric_type to a string */ diff --git a/interop/logic/utils/metric_type_ext.h b/interop/logic/utils/metric_type_ext.h index e46965514..15b33e9ef 100644 --- a/interop/logic/utils/metric_type_ext.h +++ b/interop/logic/utils/metric_type_ext.h @@ -10,7 +10,8 @@ #include "interop/constants/enums.h" #include "interop/logic/utils/enums.h" -namespace illumina { namespace interop { namespace logic { namespace utils { +namespace illumina { namespace interop { namespace logic { namespace utils +{ /** Convert metric type to metric group * @@ -97,4 +98,14 @@ namespace illumina { namespace interop { namespace logic { namespace utils { return (to_feature(type) & constants::CycleFeature) == constants::CycleFeature; } + /** Test if metric type is produced for a tile + * + * @param type metric type + * @return true if metric is produced for every tile + */ + inline bool is_tile_metric(const constants::metric_type type) + { + return (to_feature(type) & constants::TileFeature) == constants::TileFeature; + } + }}}} diff --git a/interop/model/metric_base/base_cycle_metric.h b/interop/model/metric_base/base_cycle_metric.h index b36eaf480..0731191a2 100644 --- a/interop/model/metric_base/base_cycle_metric.h +++ b/interop/model/metric_base/base_cycle_metric.h @@ -60,7 +60,6 @@ namespace illumina { namespace interop { namespace model { namespace metric_base /** Maximum cycle read **/ base_metric::uint_t m_max_cycle; }; - /** Base class for InterOp classes that contain per cycle tile specific metrics * * These classes define both a lane, tile and cycle identifier. @@ -71,7 +70,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base enum { /** Base for records written out once for each tile/cycle */ - BASE_TYPE = constants::BaseCycleType + BASE_TYPE = constants::BaseCycleType }; public: /** A cycle metric_set header @@ -86,16 +85,28 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param tile tile number * @param cycle cycle number */ - base_cycle_metric(const uint_t lane, const uint_t tile, const uint_t cycle) : + base_cycle_metric(const uint_t lane=0, const uint_t tile=0, const uint_t cycle=0) : base_metric(lane, tile), m_cycle(cycle) { } public: + /** Set id + * + * @param lane lane number + * @param tile tile number + * @param cycle cycle number + */ + void set_base(const uint_t lane, const uint_t tile, const uint_t cycle) + { + base_metric::set_base(lane, tile); + m_cycle = cycle; + } /** Set the base metric identifiers * * @param base layout base */ - void set_base(const io::layout::base_cycle_metric &base) + template + void set_base(const BaseCycleMetric &base) { base_metric::set_base(base); m_cycle = base.cycle; @@ -114,7 +125,17 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ id_t id() const { - return id(lane(), tile(), m_cycle); + return create_id(lane(), tile(), m_cycle); + } + /** Get the cycle from the unique lane/tile/cycle id + * + * @param id unique lane/tile/cycle id + * @return cycle number + */ + static id_t cycle_from_id(const id_t id) + { + // Mask tile hash (lane + tile) + return id & ~((~static_cast(0)) << TILE_BIT_SHIFT); } /** Create unique id from the lane, tile and cycle @@ -124,9 +145,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param cycle cycle number * @return unique id */ - static id_t id(const id_t lane, const id_t tile, const id_t cycle) + static id_t create_id(const id_t lane, const id_t tile, const id_t cycle) { - return base_metric::id(lane, tile) | (cycle << TILE_BIT_SHIFT); + return base_metric::create_id(lane, tile) | cycle; } private: diff --git a/interop/model/metric_base/base_metric.h b/interop/model/metric_base/base_metric.h index 328879f85..b791b27c2 100644 --- a/interop/model/metric_base/base_metric.h +++ b/interop/model/metric_base/base_metric.h @@ -9,12 +9,8 @@ */ #pragma once -#include -#include #include -#include #include "interop/util/assert.h" -#include "interop/util/type_traits.h" #include "interop/io/layout/base_metric.h" #include "interop/constants/enums.h" #include "interop/constants/typedefs.h" @@ -91,10 +87,16 @@ namespace illumina { namespace interop { namespace model { namespace metric_base typedef constants::base_tile_t base_t; enum { + TYPE=constants::UnknownMetricType, + //Compress Lane, Tile and cycle into a 64-bit Integer + // Cycle: Bits 0-32 + // Tile: Bits 32-58 + // Lane: Bits 58-64 LANE_BIT_COUNT = 6, // Supports up to 63 lanes - TILE_BIT_COUNT = 32, - LANE_BIT_SHIFT = LANE_BIT_COUNT, // Supports up to 63 lanes - TILE_BIT_SHIFT =/*LANE_BIT_COUNT+*/TILE_BIT_COUNT, // TODO change back + TILE_BIT_COUNT = 26, // Support 7 digit tile (up to 67108864) + CYCLE_BIT_COUNT = 32, // Support up to 4294967296 cycles + LANE_BIT_SHIFT = CYCLE_BIT_COUNT+TILE_BIT_COUNT, // Supports up to 63 lanes + TILE_BIT_SHIFT =CYCLE_BIT_COUNT, /** Base for records written out once for each tile */ BASE_TYPE = constants::BaseTileType, /** Tells the reader to exclude any records that have 0 for tile */ @@ -106,11 +108,21 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param lane lane number * @param tile tile number */ - base_metric(const uint_t lane, const uint_t tile) : + base_metric(const uint_t lane=0, const uint_t tile=0) : m_lane(lane), m_tile(tile) { } public: + /** Set id + * + * @param lane lane number + * @param tile tile number + */ + void set_base(const uint_t lane, const uint_t tile) + { + m_lane = lane; + m_tile = tile; + } /** Get the metric name suffix * * @return empty string @@ -122,7 +134,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * * @param base layout base */ - void set_base(const io::layout::base_metric &base) + template + void set_base(const BaseMetric &base) { m_lane = base.lane; m_tile = base.tile; @@ -134,7 +147,16 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ id_t id() const { - return id(m_lane, m_tile); + return create_id(m_lane, m_tile); + } + + /** Unique id created from both the lane and tile + * + * @return unique identifier + */ + id_t tile_hash() const + { + return create_id(m_lane, m_tile); } /** Unique id created from both the lane and tile @@ -143,9 +165,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param tile tile number * @return unique id */ - static id_t id(const id_t lane, const id_t tile, const id_t= 0)// TODO: remove hack (const id_t=0) + static id_t create_id(const id_t lane, const id_t tile, const id_t= 0)// TODO: remove hack (const id_t=0) { - return lane | (tile << LANE_BIT_SHIFT); + return (lane << LANE_BIT_SHIFT) | (tile << TILE_BIT_SHIFT); } /** Get the lane from the unique lane/tile id @@ -155,7 +177,28 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ static id_t lane_from_id(const id_t id) { - return id & ~((~0u) << LANE_BIT_SHIFT); + return id >> LANE_BIT_SHIFT; + } + /** Get the tile hash from the unique lane/tile/cycle id + * + * @param id unique lane/tile/cycle id + * @return tile hash number + */ + static id_t tile_hash_from_id(const id_t id) + { + // 1. Remove cycle information + return (id >> TILE_BIT_SHIFT) << TILE_BIT_SHIFT; + } + /** Get the tile hash from the unique lane/tile/cycle id + * + * @param id unique lane/tile/cycle id + * @return tile hash number + */ + static id_t tile_from_id(const id_t id) + { + // 1. Mask out lane + // 2. Shift tile id + return (id & ~((~static_cast(0)) << LANE_BIT_SHIFT)) >> TILE_BIT_SHIFT; } /** Lane number @@ -316,8 +359,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base const bool all_surfaces) const { - INTEROP_ASSERT(swath(method) <= swath_count); - INTEROP_ASSERT(number(method) <= tile_count); + INTEROP_ASSERTMSG(swath(method) <= swath_count, swath(method) << " <= " << swath_count); + INTEROP_ASSERTMSG(number(method) <= tile_count, number(method) << " <= " << tile_count); const uint_t column = physical_location_column(method, swath_count, all_surfaces); const uint_t row = physical_location_row(method, section_per_lane, tile_count); const uint_t row_count = section_per_lane * tile_count; @@ -378,4 +421,6 @@ namespace illumina { namespace interop { namespace model { namespace metric_base uint_t m_lane; uint_t m_tile; }; + + }}}} diff --git a/interop/model/metric_base/base_read_metric.h b/interop/model/metric_base/base_read_metric.h index 18d7911e4..0e46c5c18 100644 --- a/interop/model/metric_base/base_read_metric.h +++ b/interop/model/metric_base/base_read_metric.h @@ -52,13 +52,11 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * * These classes define both a lane, tile and read identifier. */ - class base_read_metric : public base_metric - { + class base_read_metric : public base_metric { public: - enum - { + enum { /** Base for records written out once for each tile/read */ - BASE_TYPE = constants::BaseReadType + BASE_TYPE = constants::BaseReadType }; /** A read metric_set header */ @@ -73,26 +71,39 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param read read number */ base_read_metric(const uint_t lane, const uint_t tile, const uint_t read) : - base_metric(lane, tile), m_read(read) - { } + base_metric(lane, tile), m_read(read) { } public: + + /** + * Sets ID given lane-tile-read as opposed to the BaseReadMetric + * + * @param lane lane number + * @param tile tile number + * @param read read number + */ + void set_base(const uint_t lane, const uint_t tile, const uint_t read) + { + base_metric::set_base(lane, tile); + m_read = read; + } /** Set the base metric identifiers * * @param base layout base */ - void set_base(const io::layout::base_read_metric &base) - { + template + void set_base(const BaseReadMetric &base) { base_metric::set_base(base); m_read = base.read; } + + /** Read number * * @return read number */ - uint_t read() const - { return m_read; } + uint_t read() const { return m_read; } /** Unique id created from the lane, tile and read * @@ -100,7 +111,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ id_t id() const { - return id(lane(), tile(), m_read); + return create_id(lane(), tile(), m_read); } /** Create unique id from the lane, tile and read @@ -110,9 +121,19 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param read read number * @return unique id */ - static id_t id(const id_t lane, const id_t tile, const id_t read) + static id_t create_id(const id_t lane, const id_t tile, const id_t read) + { + return base_metric::create_id(lane, tile) | read; + } + /** Get the read from the unique lane/tile/read id + * + * @param id unique lane/tile/read id + * @return read number + */ + static id_t read_from_id(const id_t id) { - return base_metric::id(lane, tile) | (read << TILE_BIT_SHIFT); + // Mask tile hash (lane + tile) + return id & ~((~static_cast(0)) << TILE_BIT_SHIFT); } private: diff --git a/interop/model/metric_base/metric_set.h b/interop/model/metric_base/metric_set.h index 18bdd18db..e8dd8f2df 100644 --- a/interop/model/metric_base/metric_set.h +++ b/interop/model/metric_base/metric_set.h @@ -41,6 +41,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base typedef std::vector metric_array_t; /** Define a write header */ typedef typename T::header_type header_type; + /** Define the base type for the metric */ + typedef typename T::base_t base_t; /** Define a metric type */ typedef T metric_type; /** Define a metric set type */ @@ -61,17 +63,28 @@ namespace illumina { namespace interop { namespace model { namespace metric_base typedef typename metric_array_t::const_iterator const_iterator; /** Metric iterator */ typedef typename metric_array_t::iterator iterator; + enum { + /** Group type enum */ + TYPE=T::TYPE + }; private: typedef std::map id_map_t; public: + /** Constructor + * + * @param version version of the file format + */ + metric_set(const ::int16_t version ) + : header_type(header_type::default_header()), m_version(version), m_data_source_exists(false) + { } /** Constructor * * @param header header information for the metric set * @param version version of the file format */ metric_set(const header_type &header = header_type::default_header(), const ::int16_t version = 0) - : header_type(header), m_version(version) + : header_type(header), m_version(version), m_data_source_exists(false) { } /** Constructor @@ -83,7 +96,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base metric_set(const metric_array_t &vec, const ::int16_t version, const header_type &header) : header_type(header), m_data(vec), - m_version(version) + m_version(version), + m_data_source_exists(false) { size_t offset = 0; for (typename metric_array_t::const_iterator b = vec.begin(), e = vec.end(); b != e; ++b) @@ -95,6 +109,22 @@ namespace illumina { namespace interop { namespace model { namespace metric_base } public: + /** Flag that indicates whether the data source exists + * + * @return true if the data source, e.g. file, exists + */ + bool data_source_exists()const + { + return !empty() || m_data_source_exists; + } + /** Set flag that indicates whether the data source exists + * + * @param exists true if the data source, e.g. file, exists + */ + void data_source_exists(const bool exists) + { + m_data_source_exists = exists; + } /** Get start of metric collection * * @return iterator to start of metric collection @@ -141,7 +171,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ size_t find(const uint_t lane, const uint_t tile, const uint_t cycle = 0) const { - return find(metric_type::id(lane, tile, cycle)); + return find(metric_type::create_id(lane, tile, cycle)); } /** Find index of metric given the id. If not found, return number of metrics @@ -164,7 +194,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ bool has_metric(const uint_t lane, const uint_t tile, const uint_t cycle = 0) const { - return has_metric(metric_type::id(lane, tile, cycle)); + return has_metric(metric_type::create_id(lane, tile, cycle)); } /** Test if set has metric @@ -214,12 +244,12 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { try { - return get_metric(metric_type::id(lane, tile, cycle)); + return get_metric(metric_type::create_id(lane, tile, cycle)); } catch (const index_out_of_bounds_exception &) { INTEROP_THROW( index_out_of_bounds_exception, "No tile available: key: " << - metric_type::id(lane, tile, cycle) << + metric_type::create_id(lane, tile, cycle) << " map: " << (m_id_map.size()) << " lane: " << (lane) << " tile: " << (tile) << @@ -371,7 +401,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ metric_array_t metrics_for_cycle(const uint_t cycle) const { - return metrics_for_cycle(cycle, (typename T::header_type *) 0); + typedef typename metric_type::base_t base_t; + return metrics_for_cycle(cycle, base_t::null()); } public: @@ -381,6 +412,12 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ size_t size() const { return m_data.size(); } + /** Test if metric set is empty + * + * @return true if metric set is empty + */ + bool empty() const + { return m_data.empty(); } /** Version of the InterOp file parsed * @@ -451,12 +488,12 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { try { - return get_metric_ref(metric_type::id(lane, tile, cycle)); + return get_metric_ref(metric_type::create_id(lane, tile, cycle)); } catch (const index_out_of_bounds_exception &) { INTEROP_THROW( index_out_of_bounds_exception,"No tile available: key: " << - metric_type::id(lane, tile, cycle) << + metric_type::create_id(lane, tile, cycle) << " map: " <<(m_id_map.size()) << " lane: " << (lane) << " tile: " << (tile) << @@ -485,7 +522,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base } private: - metric_array_t metrics_for_cycle(const uint_t cycle, base_cycle_metric_header *) const + metric_array_t metrics_for_cycle(const uint_t cycle, const constants::base_cycle_t*) const { metric_array_t cycle_metrics; cycle_metrics.reserve(size()); @@ -497,12 +534,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base return cycle_metrics; } - metric_array_t metrics_for_cycle(const uint_t, base_metric_header *) const - { - return metric_array_t(); - } - - metric_array_t metrics_for_cycle(const uint_t, base_read_metric_header *) const + metric_array_t metrics_for_cycle(const uint_t, const void *) const { return metric_array_t(); } @@ -585,6 +617,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base metric_array_t m_data; /** Version of the metric read */ ::int16_t m_version; + /** Does the file or other source exist */ + bool m_data_source_exists; // TODO: remove the following /** Map unique identifiers to the index of the metric */ diff --git a/interop/model/metrics/image_metric.h b/interop/model/metrics/image_metric.h index 987954db3..5dfdc03e3 100644 --- a/interop/model/metrics/image_metric.h +++ b/interop/model/metrics/image_metric.h @@ -38,9 +38,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics public: /** Constructor * - * @param channelCount number of channels + * @param channel_count number of channels */ - image_metric_header(ushort_t channelCount) : m_channel_count(channelCount) + image_metric_header(ushort_t channel_count) : m_channel_count(channel_count) { } /** Number of channels @@ -117,18 +117,18 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param tile tile number * @param cycle cycle number * @param channel_count number of channels - * @param minContrast minimum contrast for each channel - * @param maxContrast maximum contrast for each channel + * @param min_contrast minimum contrast for each channel + * @param max_contrast maximum contrast for each channel */ image_metric(const uint_t lane, const uint_t tile, const uint_t cycle, const ushort_t channel_count, - const ushort_array_t &minContrast, - const ushort_array_t &maxContrast) : + const ushort_array_t &min_contrast, + const ushort_array_t &max_contrast) : metric_base::base_cycle_metric(lane, tile, cycle), - m_min_contrast(minContrast), - m_max_contrast(maxContrast), + m_min_contrast(min_contrast), + m_max_contrast(max_contrast), m_channel_count(channel_count) { } @@ -140,18 +140,18 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param tile tile number * @param cycle cycle number * @param channel_count number of channels - * @param minContrast minimum contrast for each channel - * @param maxContrast maximum contrast for each channel + * @param min_contrast minimum contrast for each channel + * @param max_contrast maximum contrast for each channel */ image_metric(const uint_t lane, const uint_t tile, const uint_t cycle, const ushort_t channel_count, - const ushort_pointer_t minContrast, - const ushort_pointer_t maxContrast) : + const ushort_pointer_t min_contrast, + const ushort_pointer_t max_contrast) : metric_base::base_cycle_metric(lane, tile, cycle), - m_min_contrast(minContrast, minContrast + channel_count), - m_max_contrast(maxContrast, maxContrast + channel_count), + m_min_contrast(min_contrast, min_contrast + channel_count), + m_max_contrast(max_contrast, max_contrast + channel_count), m_channel_count(channel_count) { } diff --git a/interop/model/metrics/index_metric.h b/interop/model/metrics/index_metric.h index c9d04e88b..8fd60919f 100644 --- a/interop/model/metrics/index_metric.h +++ b/interop/model/metrics/index_metric.h @@ -32,12 +32,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ class index_info { - public: - enum - { - /** Unique type code for metric */ - TYPE = constants::Index - }; public: /** Constructor * @@ -167,6 +161,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics public: enum { + /** Unique type code for metric */ + TYPE = constants::Index, /** Latest version of the InterOp format */ LATEST_VERSION = 1 }; diff --git a/interop/model/metrics/q_by_lane_metric.h b/interop/model/metrics/q_by_lane_metric.h index 61a2e6ae2..7d9fe9ca1 100644 --- a/interop/model/metrics/q_by_lane_metric.h +++ b/interop/model/metrics/q_by_lane_metric.h @@ -14,7 +14,8 @@ #include "interop/model/metrics/q_metric.h" #include "interop/io/format/metric_format_factory.h" -namespace illumina { namespace interop { namespace model { namespace metrics { +namespace illumina { namespace interop { namespace model { namespace metrics +{ /** Total histogram by lane */ class q_by_lane_metric : public q_metric @@ -26,6 +27,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics { /** Tells the reader to include any records that have 0 for tile */ CHECK_TILE_ID = 0 }; + /** Define the base type */ + typedef constants::base_lane_t base_t; public: /** Constructor */ diff --git a/interop/model/metrics/q_collapsed_metric.h b/interop/model/metrics/q_collapsed_metric.h index 24bed397a..a02d07b2e 100644 --- a/interop/model/metrics/q_collapsed_metric.h +++ b/interop/model/metrics/q_collapsed_metric.h @@ -24,7 +24,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics { */ class q_collapsed_header : public q_score_header { - public: /** Constructor */ diff --git a/interop/model/metrics/q_metric.h b/interop/model/metrics/q_metric.h index ae75d4d86..439470b52 100644 --- a/interop/model/metrics/q_metric.h +++ b/interop/model/metrics/q_metric.h @@ -372,21 +372,21 @@ namespace illumina { namespace interop { namespace model { namespace metrics uint_t total_over_qscore(const uint_t qscore, const qscore_bin_vector_type &bins = qscore_bin_vector_type()) const { - uint_t totalCount = 0; + uint_t total_count = 0; if (bins.size() == 0) { if(qscore <= m_qscore_hist.size()) - totalCount = std::accumulate(m_qscore_hist.begin() + qscore, m_qscore_hist.end(), 0); + total_count = std::accumulate(m_qscore_hist.begin() + qscore, m_qscore_hist.end(), 0); } else { for (size_t i = 0; i < bins.size(); i++) { if (bins[i].value() >= qscore) - totalCount += m_qscore_hist[i]; + total_count += m_qscore_hist[i]; } } - return totalCount; + return total_count; } /** Number of clusters over the given q-score @@ -404,12 +404,12 @@ namespace illumina { namespace interop { namespace model { namespace metrics const qscore_bin_vector_type &bins = qscore_bin_vector_type()) const { INTEROP_ASSERT(m_qscore_hist_cumulative.size() > 0); - ::uint64_t totalCount = 0; + ::uint64_t total_count = 0; if (bins.size() == 0) { INTEROP_ASSERT(qscore > 0); if(qscore <= m_qscore_hist_cumulative.size()) - totalCount = std::accumulate(m_qscore_hist_cumulative.begin() + qscore, m_qscore_hist_cumulative.end(), + total_count = std::accumulate(m_qscore_hist_cumulative.begin() + qscore, m_qscore_hist_cumulative.end(), static_cast< ::uint64_t >(0)); } else @@ -417,10 +417,10 @@ namespace illumina { namespace interop { namespace model { namespace metrics for (size_t i = 0; i < bins.size(); i++) { if (bins[i].value() >= qscore) - totalCount += m_qscore_hist_cumulative[i]; + total_count += m_qscore_hist_cumulative[i]; } } - return totalCount; + return total_count; } /** Percent of clusters over the given q-score diff --git a/interop/model/metrics/tile_metric.h b/interop/model/metrics/tile_metric.h index c114458ef..97f0a46fa 100644 --- a/interop/model/metrics/tile_metric.h +++ b/interop/model/metrics/tile_metric.h @@ -286,7 +286,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ float percent_pf() const { - if (m_cluster_count == 0) return 0; return 100 * m_cluster_count_pf / m_cluster_count; } diff --git a/interop/model/model_exceptions.h b/interop/model/model_exceptions.h index 30b029faf..85654f5ba 100644 --- a/interop/model/model_exceptions.h +++ b/interop/model/model_exceptions.h @@ -6,16 +6,15 @@ * @copyright GNU Public License. */ #pragma once + #include -#include "interop/util/exception_specification.h" #ifdef _MSC_VER #pragma warning(disable:4290) // MSVC warns that it ignores the exception specification. #endif -namespace illumina { -namespace interop { -namespace model { +namespace illumina { namespace interop { namespace model +{ /** @defgroup model_exceptions Model Exceptions * @@ -26,83 +25,112 @@ namespace model { */ /** Exception raised if an index goes out of bounds */ - struct index_out_of_bounds_exception : public std::out_of_range { + struct index_out_of_bounds_exception : public std::out_of_range + { /** Constructor * * @param mesg error message */ - index_out_of_bounds_exception(const std::string &mesg) : std::out_of_range(mesg) { } + index_out_of_bounds_exception(const std::string &mesg) : std::out_of_range(mesg) + { } }; /** Exception raised if the channel names are invalid */ - struct invalid_channel_exception : public std::runtime_error { + struct invalid_channel_exception : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_channel_exception(const std::string &mesg) : std::runtime_error(mesg) { } + invalid_channel_exception(const std::string &mesg) : std::runtime_error(mesg) + { } }; /** Exception raised if the read was not found */ - struct invalid_read_exception : public std::runtime_error { + struct invalid_read_exception : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_read_exception(const std::string &mesg) : std::runtime_error(mesg) { } + invalid_read_exception(const std::string &mesg) : std::runtime_error(mesg) + { } }; /** Exception raised if the metric type is invalid */ - struct invalid_metric_type : public std::invalid_argument { + struct invalid_metric_type : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_metric_type(const std::string &mesg) : std::invalid_argument(mesg) { } + invalid_metric_type(const std::string &mesg) : std::runtime_error(mesg) + { } }; + /** Exception raised if the filter options are invalid */ - struct invalid_filter_option : public std::invalid_argument { + struct invalid_filter_option : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_filter_option(const std::string &mesg) : std::invalid_argument(mesg) { } + invalid_filter_option(const std::string &mesg) : std::runtime_error(mesg) + { } }; + /** Exception raised if the tile naming convention is invalid */ - struct invalid_tile_naming_method : public std::invalid_argument { + struct invalid_tile_naming_method : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_tile_naming_method(const std::string &mesg) : std::invalid_argument(mesg) { } + invalid_tile_naming_method(const std::string &mesg) : std::runtime_error(mesg) + { } }; + /** Exception raised if a function is given an invalid parameter */ - struct invalid_parameter : public std::invalid_argument { + struct invalid_parameter : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_parameter(const std::string &mesg) : std::invalid_argument(mesg) { } + invalid_parameter(const std::string &mesg) : std::runtime_error(mesg) + { } }; /** Exception raised if the column type is invalid */ - struct invalid_column_type : public std::runtime_error { + struct invalid_column_type : public std::runtime_error + { + /** Constructor + * + * @param mesg error message + */ + invalid_column_type(const std::string &mesg) : std::runtime_error(mesg) + { } + }; + + /** Exception raised if the RunInfo does not match the InterOps + */ + struct invalid_run_info_exception : public std::runtime_error + { /** Constructor * * @param mesg error message */ - invalid_column_type(const std::string &mesg) : std::runtime_error(mesg) { } + invalid_run_info_exception(const std::string &mesg) : std::runtime_error(mesg) + { } }; /** @} */ -} -} -} +}}} diff --git a/interop/model/plot/candle_stick_point.h b/interop/model/plot/candle_stick_point.h index 268a85a7b..3aea65a03 100644 --- a/interop/model/plot/candle_stick_point.h +++ b/interop/model/plot/candle_stick_point.h @@ -8,6 +8,7 @@ #pragma once #include +#include "interop/util/math.h" #include "interop/model/plot/data_point.h" namespace illumina { namespace interop { namespace model { namespace plot diff --git a/interop/model/plot/filter_options.h b/interop/model/plot/filter_options.h index 6629983fb..530d5ad47 100644 --- a/interop/model/plot/filter_options.h +++ b/interop/model/plot/filter_options.h @@ -153,7 +153,8 @@ namespace illumina { namespace interop { namespace model { namespace plot template bool valid_tile_cycle(const Metric &metric) const { - return valid_tile(metric) && valid_cycle(metric, typename Metric::base_t()); + typedef typename Metric::base_t base_t; + return valid_tile(metric) && valid_cycle(metric, base_t::null()); } /** Test if metric is a valid read @@ -164,7 +165,8 @@ namespace illumina { namespace interop { namespace model { namespace plot template bool valid_read(const Metric &metric) const { - return valid_read(metric, typename Metric::base_t()); + typedef typename Metric::base_t base_t; + return valid_read(metric, base_t::null()); } /** Test if all channels were requested @@ -553,29 +555,21 @@ namespace illumina { namespace interop { namespace model { namespace plot private: template - bool valid_cycle(const Metric &metric, constants::base_cycle_t) const + bool valid_cycle(const Metric &metric, const constants::base_cycle_t*) const { - return metric.cycle() == m_cycle; + return m_cycle == static_cast(ALL_IDS) || metric.cycle() == m_cycle; } template - bool valid_cycle(const Metric &, constants::base_tile_t) const + bool valid_cycle(const Metric &, const void*) const { return true; } template - bool valid_cycle(const Metric &, constants::base_read_t) const - { return true; } - - template - bool valid_read(const Metric &metric, constants::base_read_t) const - { return metric.read() == m_read; } - - template - bool valid_read(const Metric &, constants::base_cycle_t) const - { return true; } + bool valid_read(const Metric &metric, const constants::base_read_t*) const + { return m_read == static_cast(ALL_IDS) || metric.read() == m_read; } template - bool valid_read(const Metric &, constants::base_tile_t) const + bool valid_read(const Metric &, const void*) const { return true; } diff --git a/interop/model/plot/flowcell_data.h b/interop/model/plot/flowcell_data.h index 85fc98f8e..1e4f449e3 100644 --- a/interop/model/plot/flowcell_data.h +++ b/interop/model/plot/flowcell_data.h @@ -6,160 +6,264 @@ * @copyright GNU Public License. */ #pragma once + #include "interop/util/exception.h" #include "interop/util/assert.h" #include "interop/model/plot/series.h" #include "interop/model/plot/axes.h" #include "interop/model/plot/heatmap_data.h" -namespace illumina { namespace interop { namespace model { namespace plot { - -/** Encapsulates all data for a flowcell heat map - * - * @todo refactor so it does not depend on 2-axes (from heatmap/chart_data) - */ -class flowcell_data : public heatmap_data +namespace illumina { namespace interop { namespace model { namespace plot { -public: - /** Constructor */ - flowcell_data() : m_swath_count(0), m_tile_count(0){} -public: - /** Resize the flowcell heat map to the given number of rows and columns - * - * @param lanes number of lanes - * @param swaths number of swaths - * @param tiles number of tiles - */ - void resize(const size_t lanes, const size_t swaths, const size_t tiles) - { - m_swath_count = swaths; - m_tile_count = tiles; - heatmap_data::resize(lanes, swaths*tiles); - m_data.resize(heatmap_data::length()); - } - /** Clear the data - */ - void clear() - { - heatmap_data::clear(); - m_data.clear(); - } - /** Set data at given location in the flowcell - * - * @param lane_idx lane index - * @param loc physical tile location - * @param tile_id id of the tile - * @param value value of the metric - */ - void set_data(const size_t lane_idx, const size_t loc, const ::uint32_t tile_id, const float value) - throw(model::index_out_of_bounds_exception) - { - if(lane_idx >= lane_count()) - INTEROP_THROW( model::index_out_of_bounds_exception, "Lane index out of bounds"); - if(loc >= column_count()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Location index out of bounds"); - //INTEROP_ASSERT(lane < m_data.size()); - heatmap_data::operator()(lane_idx, loc) = value; - m_data[index_of(lane_idx, loc)] = tile_id; - } - /** Get the tile id associated with the location - * - * @param lane_idx - * @param loc - * @return tile id - */ - ::uint32_t tile_id(const size_t lane_idx, const size_t loc)const throw(model::index_out_of_bounds_exception) - { - if(lane_idx >= lane_count()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Lane index out of bounds"); - if(loc >= column_count()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Location index out of bounds"); - INTEROP_ASSERT(index_of(lane_idx, loc) < m_data.size()); - return m_data[index_of(lane_idx, loc)]; - } - /** Set the axis - * - * @param plot_axis single axis - */ - void set_saxis(const plot::axis& plot_axis) - { - set_yaxis(plot_axis); - } - /** Set the label of the axis - * - * @param label text label - */ - void set_label(const std::string& label) - { - set_ylabel(label); - } - /** Get the single axis - * - * @return single axis - */ - const plot::axis& saxis()const - { - return y_axis(); - } - /** Set the sub title - * - * @param subtitle label string - */ - void set_subtitle(const std::string& subtitle) - { - m_subtitle = subtitle; - } - /** Set the limits of the axis - * - * @param vmin minimum value - * @param vmax maximum value - */ - void set_range(const float vmin, const float vmax) - { - set_yrange(vmin, vmax); - } - /** Get the sub title - * - * @return sub title - */ - const std::string& subtitle()const - { - return m_subtitle; - } - /** Get number of lanes - * - * @return number of lanes - */ - size_t lane_count()const - { - return row_count(); - } - /** Get number of swaths - * - * @return number of swaths - */ - size_t swath_count()const - { - return m_swath_count; - } - /** Get number of tiles + + /** Encapsulates all data for a flowcell heat map * - * @return number of tiles + * @todo refactor so it does not depend on 2-axes (from heatmap/chart_data) */ - size_t tile_count()const + class flowcell_data : public heatmap_data { - return m_tile_count; - } + public: + /** Constructor */ + flowcell_data() : m_data(0), m_swath_count(0), m_tile_count(0), m_free(false) + { } + + /** Destructor */ + virtual ~flowcell_data() + { + clear(); + } + + public: + /** Resize the heat map to the given number of rows and columns + * + * @param data_buffer buffer to hold the flow cell values + * @param id_buffer buffer to hold the tile ids + * @param lanes number of lanes + * @param swaths number of swaths + * @param tiles number of tiles + */ + void set_buffer(float *data_buffer, + ::uint32_t *id_buffer, + const size_t lanes, + const size_t swaths, + const size_t tiles) + { + heatmap_data::set_buffer(data_buffer, lanes, swaths * tiles); + set_buffer(id_buffer, swaths, tiles); + } + + /** Resize the flowcell heat map to the given number of rows and columns + * + * @param lanes number of lanes + * @param swaths number of swaths + * @param tiles number of tiles + */ + void resize(const size_t lanes, const size_t swaths, const size_t tiles) + { + if (lanes != row_count() && swaths != m_swath_count && tiles != m_tile_count) + { + heatmap_data::resize(lanes, swaths * tiles); + resize(swaths, tiles); + } + } + + /** Clear the data + */ + void clear() + { + heatmap_data::clear(); + if (m_free) + { + delete[] m_data; + m_data = 0; + m_free = false; + } + m_swath_count = 0; + m_tile_count = 0; + } + + /** Set data at given location in the flowcell + * + * @param lane_idx lane index + * @param loc physical tile location + * @param tile_id id of the tile + * @param value value of the metric + */ + void set_data(const size_t lane_idx, const size_t loc, const ::uint32_t tile_id, const float value) + throw(model::index_out_of_bounds_exception) + { + if (lane_idx >= lane_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Lane index out of bounds"); + if (loc >= column_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Location index out of bounds"); + heatmap_data::operator()(lane_idx, loc) = value; + INTEROP_ASSERT(m_data != 0); + m_data[index_of(lane_idx, loc)] = tile_id; + } + + /** Get the tile id associated with the location + * + * @param lane_idx + * @param loc + * @return tile id + */ + ::uint32_t tile_id(const size_t lane_idx, const size_t loc) const throw(model::index_out_of_bounds_exception) + { + if (lane_idx >= lane_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Lane index out of bounds"); + if (loc >= column_count()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Location index out of bounds"); + INTEROP_ASSERT(index_of(lane_idx, loc) < length()); + INTEROP_ASSERT(m_data != 0); + return m_data[index_of(lane_idx, loc)]; + } + + /** Set the axis + * + * @param plot_axis single axis + */ + void set_saxis(const plot::axis &plot_axis) + { + set_yaxis(plot_axis); + } + + /** Set the label of the axis + * + * @param label text label + */ + void set_label(const std::string &label) + { + set_ylabel(label); + } + + /** Get the single axis + * + * @return single axis + */ + const plot::axis &saxis() const + { + return y_axis(); + } + + /** Set the sub title + * + * @param subtitle label string + */ + void set_subtitle(const std::string &subtitle) + { + m_subtitle = subtitle; + } + + /** Set the limits of the axis + * + * @param vmin minimum value + * @param vmax maximum value + */ + void set_range(const float vmin, const float vmax) + { + set_yrange(vmin, vmax); + } + + /** Get the sub title + * + * @return sub title + */ + const std::string &subtitle() const + { + return m_subtitle; + } + + /** Get number of lanes + * + * @return number of lanes + */ + size_t lane_count() const + { + return row_count(); + } + + /** Get number of swaths + * + * @return number of swaths + */ + size_t swath_count() const + { + return m_swath_count; + } + + /** Get number of tiles + * + * @return number of tiles + */ + size_t tile_count() const + { + return m_tile_count; + } + + /** Get total number of tiles for a single lane + * + * @return total number of tiles for a single lane + */ + size_t total_tile_count() const + { + return m_tile_count * m_swath_count; + } + + protected: + /** Resize the heat map to the given number of rows and columns + * + * @param id_buffer buffer to hold the tile ids + */ + void set_buffer(::uint32_t *id_buffer) throw(invalid_parameter) + { + if (m_free) throw invalid_parameter("Cannot use internal buffer map with external buffer"); + if (empty()) throw invalid_parameter("Cannot set external buffer to empty map"); + m_data = id_buffer; + } + + /** Resize the heat map to the given number of rows and columns + * + * @param id_buffer buffer to hold the tile ids + * @param swaths number of swaths + * @param tiles number of tiles + */ + void set_buffer(::uint32_t *id_buffer, + const size_t swaths, + const size_t tiles) + { + if (m_free) delete[] m_data; + m_data = id_buffer; + m_swath_count = swaths; + m_tile_count = tiles; + m_free = false; + } + + /** Resize the tile ID map to the given number of rows and columns + * + * @param swaths number of swaths + * @param tiles number of tiles + */ + void resize(const size_t swaths, + const size_t tiles) + { + if (m_free) delete[] m_data; + m_swath_count = swaths; + m_tile_count = tiles; + m_data = new ::uint32_t[heatmap_data::length()]; + std::memset(reinterpret_cast(m_data), 0, sizeof(::uint32_t) * heatmap_data::length()); + m_free = true; + } protected: /** Array of tile numbers for each tile */ - std::vector< ::uint32_t > m_data; + ::uint32_t* m_data; /** Sub title */ std::string m_subtitle; /** Number of swaths per lane */ size_t m_swath_count; /** Number of tiles per swath */ size_t m_tile_count; + bool m_free; }; }}}} diff --git a/interop/model/plot/heatmap_data.h b/interop/model/plot/heatmap_data.h index bafdbc2db..3b23b4418 100644 --- a/interop/model/plot/heatmap_data.h +++ b/interop/model/plot/heatmap_data.h @@ -6,138 +6,200 @@ * @copyright GNU Public License. */ #pragma once + #include "interop/util/assert.h" #include "interop/util/exception.h" #include "interop/model/plot/series.h" #include "interop/model/plot/axes.h" #include "interop/model/plot/chart_data.h" -namespace illumina { namespace interop { namespace model { namespace plot { - -/** Encapsulates all data for a heatmap - */ -class heatmap_data : public chart_data +namespace illumina { namespace interop { namespace model { namespace plot { -public: - /** Constructor */ - heatmap_data() : m_num_columns(0), m_num_rows(0){} - -public: - /** Resize the heat map to the given number of rows and columns - * - * @param rows number of rows - * @param cols number of columns - */ - void resize(const size_t rows, const size_t cols) - { - m_data.resize(rows*cols); - m_num_columns = cols; - m_num_rows = rows; - } - /** Get value at given row and column - * - * TODO: This should thrown an exception if wrong - * - * @param row row index - * @param col column index - * @return value - */ - float at(const size_t row, const size_t col)const throw(model::index_out_of_bounds_exception) - { - if(row >= m_num_rows) - INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); - if(col >= m_num_columns) - INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds"); - const size_t idx = index_of(row, col); - INTEROP_ASSERT(idx < m_data.size()); - return m_data[idx]; - } - /** Get value at given row and column - * - * TODO: This should thrown an exception if wrong - * - * @param row row index - * @param col column index - * @return value + /** Encapsulates all data for a heatmap */ - float& operator()(const size_t row, const size_t col) throw(model::index_out_of_bounds_exception) + class heatmap_data : public chart_data { - if(row >= m_num_rows) - INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); - if(col >= m_num_columns) - INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds"); - const size_t idx = index_of(row, col); - INTEROP_ASSERT(idx < m_data.size()); - return m_data[idx]; - } - /** Get value at given row and column - * - * TODO: This should thrown an exception if wrong - * - * @param row row index - * @param col column index - * @return value - */ - float operator()(const size_t row, const size_t col)const throw(model::index_out_of_bounds_exception) - { - if(row >= m_num_rows) - INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); - if(col >= m_num_columns) - INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds"); - const size_t idx = index_of(row, col); - INTEROP_ASSERT(idx < m_data.size()); - return m_data[idx]; - } - /** Get number of rows - * - * @return number of rows - */ - size_t row_count()const - { - return m_num_rows; - } - /** Get number of columns - * - * @return number of columns - */ - size_t column_count()const - { - return m_num_columns; - } - /** Get number of rows * columns - * - * @return number of rows * columns - */ - size_t length()const - { - return m_num_columns*m_num_rows; - } - /** Clear the data - */ - void clear() - { - m_data.clear(); - m_num_columns = 0; - m_num_rows = 0; - } -protected: - /** Get the index of the row and column in the array - * - * @param row row index - * @param col column index - * @return index in array - */ - size_t index_of(const size_t row, const size_t col)const - { - return row*m_num_columns + col; - } + public: + /** Constructor */ + heatmap_data() : m_data(0), m_num_columns(0), m_num_rows(0), m_free(false) + { } + /** Destructor */ + virtual ~heatmap_data() + { + clear(); + } + + public: + /** Resize the heat map to the given number of rows and columns + * + * @param data use the given buffer to back the heat mapll + */ + void set_buffer(float* data) throw(invalid_parameter) + { + if(m_free) throw invalid_parameter("Cannot use internal buffer map with external buffer"); + if(empty()) throw invalid_parameter("Cannot set external buffer to empty map"); + m_data = data; + } + /** Resize the heat map to the given number of rows and columns + * + * @param data use the given buffer to back the heat map + * @param rows number of rows + * @param cols number of columns + */ + void set_buffer(float* data, const size_t rows, const size_t cols) + { + if(m_free) delete[] m_data; + m_data = data; + m_num_columns = cols; + m_num_rows = rows; + m_free=false; + } + /** Resize the heat map to the given number of rows and columns + * + * @param rows number of rows + * @param cols number of columns + */ + void resize(const size_t rows, const size_t cols) + { + if(rows != m_num_rows && cols != m_num_columns) + { + if (m_free) delete[] m_data; + m_data = new float[cols * rows]; + std::memset(reinterpret_cast(m_data), 0, sizeof(float)*rows*cols); + m_num_columns = cols; + m_num_rows = rows; + m_free = true; + } + } + + /** Get value at given row and column + * + * TODO: This should thrown an exception if wrong + * + * @param row row index + * @param col column index + * @return value + */ + float at(const size_t row, const size_t col) const throw(model::index_out_of_bounds_exception) + { + if (row >= m_num_rows) + INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); + if (col >= m_num_columns) + INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds"); + const size_t idx = index_of(row, col); + INTEROP_ASSERT(idx < m_num_rows*m_num_columns); + INTEROP_ASSERT(m_data != 0); + return m_data[idx]; + } + + /** Get value at given row and column + * + * TODO: This should thrown an exception if wrong + * + * @param row row index + * @param col column index + * @return value + */ + float &operator()(const size_t row, const size_t col) throw(model::index_out_of_bounds_exception) + { + if (row >= m_num_rows) + INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); + if (col >= m_num_columns) + INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds"); + const size_t idx = index_of(row, col); + INTEROP_ASSERT(idx < m_num_rows*m_num_columns); + INTEROP_ASSERT(m_data != 0); + return m_data[idx]; + } + + /** Get value at given row and column + * + * TODO: This should thrown an exception if wrong + * + * @param row row index + * @param col column index + * @return value + */ + float operator()(const size_t row, const size_t col) const throw(model::index_out_of_bounds_exception) + { + if (row >= m_num_rows) + INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); + if (col >= m_num_columns) + INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds"); + const size_t idx = index_of(row, col); + INTEROP_ASSERT(idx < m_num_rows*m_num_columns); + INTEROP_ASSERT(m_data != 0); + return m_data[idx]; + } + + /** Get number of rows + * + * @return number of rows + */ + size_t row_count() const + { + return m_num_rows; + } + + /** Get number of columns + * + * @return number of columns + */ + size_t column_count() const + { + return m_num_columns; + } + + /** Get number of rows * columns + * + * @return number of rows * columns + */ + size_t length() const + { + return m_num_columns * m_num_rows; + } + /** Test if heatmap is empty + * + * @return true if there are not values in heatmap + */ + bool empty()const + { + return length()==0; + } + + /** Clear the data + */ + void clear() + { + if(m_free) + { + delete[] m_data; + m_data=0; + m_free = false; + } + m_num_columns = 0; + m_num_rows = 0; + } + + protected: + /** Get the index of the row and column in the array + * + * @param row row index + * @param col column index + * @return index in array + */ + size_t index_of(const size_t row, const size_t col) const + { + return row * m_num_columns + col; + } -private: - std::vector m_data; - size_t m_num_columns; - size_t m_num_rows; -}; + private: + float* m_data; + size_t m_num_columns; + size_t m_num_rows; + bool m_free; + }; }}}} diff --git a/interop/model/run/cycle_range.h b/interop/model/run/cycle_range.h index 483d8d712..57c6614b6 100644 --- a/interop/model/run/cycle_range.h +++ b/interop/model/run/cycle_range.h @@ -7,7 +7,6 @@ */ #pragma once -#include #include #include "interop/util/assert.h" #include "interop/io/format/generic_layout.h" diff --git a/interop/model/run/image_dimensions.h b/interop/model/run/image_dimensions.h index 173d54775..aabe6dafd 100644 --- a/interop/model/run/image_dimensions.h +++ b/interop/model/run/image_dimensions.h @@ -7,16 +7,11 @@ * @copyright GNU Public License. */ #pragma once + #include #include -namespace illumina -{ -namespace interop -{ -namespace model -{ -namespace run +namespace illumina { namespace interop { namespace model { namespace run { /** Dimenions of the image * @@ -29,7 +24,8 @@ namespace run * @param width width of the image * @param height height of the image */ - image_dimensions(size_t width=0, size_t height=0) : m_width(width), m_height(height){} + image_dimensions(size_t width = 0, size_t height = 0) : m_width(width), m_height(height) + {} public: /** @defgroup image_dimensions Image Dimensions @@ -44,21 +40,22 @@ namespace run * * @return image width */ - size_t width()const{return m_width;} + size_t width() const + { return m_width; } + /** Get the height of the image * * @return image height */ - size_t height()const{return m_height;} + size_t height() const + { return m_height; } /** @} */ private: size_t m_width; size_t m_height; + friend class info; }; -} -} -} -} +}}}} diff --git a/interop/model/run/info.h b/interop/model/run/info.h index 9e419b989..c2bc56adb 100644 --- a/interop/model/run/info.h +++ b/interop/model/run/info.h @@ -9,6 +9,7 @@ * @copyright GNU Public License. */ #pragma once + #include #include #include "interop/util/exception.h" @@ -24,212 +25,270 @@ #pragma warning(disable:4290) // MSVC warns that it ignores the exception specification. #endif -namespace illumina { namespace interop { namespace model { namespace run { - -/** Metadata describing parameters that can change between runs - * - * This class parses the RunInfo.xml XML file - */ -class info +namespace illumina { namespace interop { namespace model { namespace run { -public: - /** Unsigned integer type */ - typedef ::uint32_t uint_t; - /** String vector type */ - typedef std::vector str_vector_t; - /** Read vector type */ - typedef std::vector< read_info> read_vector_t; -public: - /** Constructor - * - * @param name name of the run - * @param date date of the run - * @param version xml file format version - * @param flowcell layout of the flowcell - * @param channels names of the color channels - * @param image_dim dimensions of the image - * @param reads description of the reads - */ - info(const std::string& name="", - const std::string& date="", - const uint_t version=0, - const flowcell_layout& flowcell=flowcell_layout(), - const str_vector_t& channels=str_vector_t(), - const image_dimensions& image_dim=image_dimensions(), - const read_vector_t& reads=read_vector_t()) : - m_name(name), - m_date(date), - m_version(version), - m_flowcell(flowcell), - m_channels(channels), - m_image_dim(image_dim), - m_reads(reads) - { - } - /** Constructor + + /** Metadata describing parameters that can change between runs * - * @param flowcell layout of the flowcell - * @param channels string list of channel names - * @param reads description of the reads + * This class parses the RunInfo.xml XML file */ - info(const flowcell_layout& flowcell, - const str_vector_t& channels, - const read_vector_t& reads) : - m_version(0), - m_flowcell(flowcell), - m_channels(channels), - m_reads(reads) + class info { - } + public: + /** Unsigned integer type */ + typedef ::uint32_t uint_t; + /** String vector type */ + typedef std::vector str_vector_t; + /** Read vector type */ + typedef std::vector read_vector_t; + /** Constant read iterator type */ + typedef read_vector_t::const_iterator const_read_iterator; + public: + /** Constructor + * + * @param name name of the run + * @param date date of the run + * @param version xml file format version + * @param flowcell layout of the flowcell + * @param channels names of the color channels + * @param image_dim dimensions of the image + * @param reads description of the reads + */ + info(const std::string &name = "", + const std::string &date = "", + const uint_t version = 0, + const flowcell_layout &flowcell = flowcell_layout(), + const str_vector_t &channels = str_vector_t(), + const image_dimensions &image_dim = image_dimensions(), + const read_vector_t &reads = read_vector_t()) : + m_name(name), + m_date(date), + m_version(version), + m_flowcell(flowcell), + m_channels(channels), + m_image_dim(image_dim), + m_reads(reads), + m_total_cycle_count(0) + { + m_total_cycle_count = total_cycles(); + } -public: - /** @defgroup run_info Run Info - * - * Information describing the run - * - * @ingroup run_metrics - * @ref illumina::interop::model::run::info "See full class description" - * @{ - */ + /** Constructor + * + * @param flowcell layout of the flowcell + * @param channels string list of channel names + * @param reads description of the reads + */ + info(const flowcell_layout &flowcell, + const str_vector_t &channels, + const read_vector_t &reads) : + m_version(0), + m_flowcell(flowcell), + m_channels(channels), + m_reads(reads), + m_total_cycle_count(0) + { + m_total_cycle_count = total_cycles(); + } - /** Read run information from run folder - * - * @param run_folder run folder containing RunInfo.xml - */ - void read(const std::string& run_folder) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception); - -public: - /** Get the name of the run - * - * @return name of the run - */ - const std::string& name()const{return m_name;} - /** Get the date of the run - * - * @return date of the run - */ - const std::string& date()const{return m_date;} - /** Get the version of the XML file format - * - * @return version of the XML file format - */ - uint_t version()const{return m_version;} - /** Get the layout of the flowcell - * - * @return layout of the flowcell - */ - const flowcell_layout& flowcell()const{ return m_flowcell;} - /** Get the channel labels - * - * @return channel labels - */ - const str_vector_t& channels()const{ return m_channels;} - /** Get the dimensions of the tile image - * - * @return tile image dimensions - */ - const image_dimensions& dimensions_of_image()const{ return m_image_dim;} - /** Get the read info - * - * @return reads - */ - const read_vector_t & reads()const{return m_reads;} - /** Check if there is an index read - * - * @return true if there is an index read - */ - bool is_indexed()const - { - for(read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end();b != e;++b) - if(b->is_index()) return true; - return false; - } - /** Get read with given number - * - * @param read_number number of the read - * @return read - */ - const read_info& read(const size_t read_number)const - { - for(read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end();b != e;++b) - if(b->number() == read_number) return *b; - INTEROP_THROW( invalid_read_exception, "Read number not found: " << read_number); - } - /** Set the channel labels - * - * @param channels channel labels - */ - void channels(const str_vector_t& channels){ m_channels = channels;} - /** Set the tile naming method - * - * @param naming_method tile naming method - */ - void set_naming_method(const constants::tile_naming_method naming_method) - { - m_flowcell.set_naming_method(naming_method); - } - /** Get total number of cycles - * - * @return total number of cycles - */ - size_t total_cycles()const - { - size_t total = 0; - for(read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end();b != e;++b) - total += b->total_cycles(); - return total; - } - /** Get usable number of cycles - * - * @return usable number of cycles - */ - size_t useable_cycles()const - { - size_t total = 0; - for(read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end();b != e;++b) - total += b->useable_cycles(); - return total; - } - /** @} */ - -public: - /** Read run information from the given XML file - * - * @param filename xml file - */ - void read_file(const std::string& filename) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception); - /** String containing xml data - * - * @param data xml string data - */ - void parse(char* data) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception); - -private: - std::string m_name; - std::string m_date; - uint_t m_version; - flowcell_layout m_flowcell; - str_vector_t m_channels; - image_dimensions m_image_dim; - read_vector_t m_reads; -}; - -} -} -} -} + public: + /** @defgroup run_info Run Info + * + * Information describing the run + * + * @ingroup run_metrics + * @ref illumina::interop::model::run::info "See full class description" + * @{ + */ + + /** Read run information from run folder + * + * @param run_folder run folder containing RunInfo.xml + */ + void read(const std::string &run_folder) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception); + + /** Test if tile list matches flowcell layout + * + * @throws invalid_run_info_exception + */ + void validate()const throw(invalid_run_info_exception,invalid_tile_naming_method); + /** Test if tile list matches flowcell layout + * + * @param lane lane number + * @param tile tile number + * @throws invalid_run_info_exception + */ + void validate(const ::uint32_t lane, const ::uint32_t tile)const throw(invalid_run_info_exception); + /** Test if tile list matches flowcell layout + * + * @param lane lane number + * @param tile tile number + * @param cycle cycle number + * @throws invalid_run_info_exception + */ + void validate_cycle(const ::uint32_t lane, const ::uint32_t tile, const size_t cycle)const + throw(invalid_run_info_exception); + /** Test if tile list matches flowcell layout + * + * @param lane lane number + * @param tile tile number + * @param read read number + * @throws invalid_run_info_exception + */ + void validate_read(const ::uint32_t lane, const ::uint32_t tile, const size_t read)const + throw(invalid_run_info_exception); + + public: + /** Get the name of the run + * + * @return name of the run + */ + const std::string &name() const + { return m_name; } + + /** Get the date of the run + * + * @return date of the run + */ + const std::string &date() const + { return m_date; } + + /** Get the version of the XML file format + * + * @return version of the XML file format + */ + uint_t version() const + { return m_version; } + + /** Get the layout of the flowcell + * + * @return layout of the flowcell + */ + const flowcell_layout &flowcell() const + { return m_flowcell; } + + /** Get the channel labels + * + * @return channel labels + */ + const str_vector_t &channels() const + { return m_channels; } + + /** Get the dimensions of the tile image + * + * @return tile image dimensions + */ + const image_dimensions &dimensions_of_image() const + { return m_image_dim; } + + /** Get the read info + * + * @return reads + */ + const read_vector_t &reads() const + { return m_reads; } + + /** Check if there is an index read + * + * @return true if there is an index read + */ + bool is_indexed() const + { + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + if (b->is_index()) return true; + return false; + } + + /** Get read with given number + * + * @param read_number number of the read + * @return read + */ + const read_info &read(const size_t read_number) const + { + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + if (b->number() == read_number) return *b; + INTEROP_THROW(invalid_read_exception, "Read number not found: " << read_number); + } + + /** Set the channel labels + * + * @param channels channel labels + */ + void channels(const str_vector_t &channels) + { m_channels = channels; } + + /** Set the tile naming method + * + * @param naming_method tile naming method + */ + void set_naming_method(const constants::tile_naming_method naming_method) + { + m_flowcell.set_naming_method(naming_method); + } + + /** Get total number of cycles + * + * @return total number of cycles + */ + size_t total_cycles() const + { + size_t total = 0; + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + total += b->total_cycles(); + return total; + } + + /** Get usable number of cycles + * + * @return usable number of cycles + */ + size_t useable_cycles() const + { + size_t total = 0; + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + total += b->useable_cycles(); + return total; + } + /** @} */ + + public: + /** Read run information from the given XML file + * + * @param filename xml file + */ + void read_file(const std::string &filename) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception); + + /** String containing xml data + * + * @param data xml string data + */ + void parse(char *data) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception); + + private: + std::string m_name; + std::string m_date; + uint_t m_version; + flowcell_layout m_flowcell; + str_vector_t m_channels; + image_dimensions m_image_dim; + read_vector_t m_reads; + size_t m_total_cycle_count; + }; + +}}}} #ifdef _MSC_VER #pragma warning(pop) diff --git a/interop/model/run/parameters.h b/interop/model/run/parameters.h index f934a40b5..1f7a33b61 100644 --- a/interop/model/run/parameters.h +++ b/interop/model/run/parameters.h @@ -9,6 +9,7 @@ * @copyright GNU Public License. */ #pragma once + #include #include #include "interop/util/exception.h" @@ -20,101 +21,97 @@ #pragma warning(disable:4290)// MSVC warns that it ignores the exception specification. #endif -namespace illumina -{ -namespace interop -{ -namespace model -{ -namespace run +namespace illumina { namespace interop { namespace model { namespace run { -/** Metadata describing sequencing run - * - * This class parses the RunParameters.xml XML file. It only provides - * the instrument type and version of the XML file format. We do not plan - * to support parsing any other values in this file. - * - * This class is only used on legacy platforms where: - * 1. The channel names were not given in the RunInfo.xml - * 2. The bins were not provided in the header of the QMetrics.bin (v4) - * - * We do not support parsing this file on later platforms. - */ -class parameters -{ -public: - /** Unsigned integer type */ - typedef ::uint32_t uint_t; - /** Instrument type enum */ - typedef constants::instrument_type instrument_type_t; - -public: - /** Constructor + /** Metadata describing sequencing run * - * @param version XML format version - * @param instrument_type type code of the instrument - */ - parameters(const uint_t version=0, - const instrument_type_t instrument_type=constants::UnknownInstrument - ) : - m_version(version), - m_instrument_type(instrument_type) - { - } - -public: - /** Get the type of instrument + * This class parses the RunParameters.xml XML file. It only provides + * the instrument type and version of the XML file format. We do not plan + * to support parsing any other values in this file. * - * @return type of instrument - */ - instrument_type_t instrument_type()const{return m_instrument_type;} - /** Get the version of the xml format + * This class is only used on legacy platforms where: + * 1. The channel names were not given in the RunInfo.xml + * 2. The bins were not provided in the header of the QMetrics.bin (v4) * - * @return version of the xml format + * We do not support parsing this file on later platforms. */ - uint_t version()const{return m_version;} + class parameters + { + public: + /** Unsigned integer type */ + typedef ::uint32_t uint_t; + /** Instrument type enum */ + typedef constants::instrument_type instrument_type_t; -public: - /** Read run metadata from the given run folder - * - * @param run_folder run folder containing RunParameters.xml - */ - void read(const std::string& run_folder) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception); - /** Read run metadata from the given XML file - * - * @param filename xml file - */ - void read_file(const std::string& filename) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception); - /** String containing xml data - * - * @param data xml string data - */ - void parse(char* data) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception); + public: + /** Constructor + * + * @param version XML format version + * @param instrument_type type code of the instrument + */ + parameters(const uint_t version = 0, + const instrument_type_t instrument_type = constants::UnknownInstrument + ) : + m_version(version), + m_instrument_type(instrument_type) + { + } + + public: + /** Get the type of instrument + * + * @return type of instrument + */ + instrument_type_t instrument_type() const + { return m_instrument_type; } + + /** Get the version of the xml format + * + * @return version of the xml format + */ + uint_t version() const + { return m_version; } + + public: + /** Read run metadata from the given run folder + * + * @param run_folder run folder containing RunParameters.xml + */ + void read(const std::string &run_folder) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception); + + /** Read run metadata from the given XML file + * + * @param filename xml file + */ + void read_file(const std::string &filename) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception); + + /** String containing xml data + * + * @param data xml string data + */ + void parse(char *data) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception); -private: - void set_instrument_id(std::string& application_name, std::string& multi_surface); + private: + void set_instrument_id(std::string &application_name, std::string &multi_surface); -private: - uint_t m_version; - instrument_type_t m_instrument_type; -}; -} -} -} -} + private: + uint_t m_version; + instrument_type_t m_instrument_type; + }; +}}}} #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/interop/model/run/read_info.h b/interop/model/run/read_info.h index b2137a338..4928befee 100644 --- a/interop/model/run/read_info.h +++ b/interop/model/run/read_info.h @@ -6,89 +6,95 @@ * @copyright GNU Public License. */ #pragma once + #include #include "interop/model/run/cycle_range.h" #include "interop/io/format/generic_layout.h" -namespace illumina { namespace interop { namespace model { namespace run { - -/** Defines a range over cycles - * - */ -class read_info : public cycle_range +namespace illumina { namespace interop { namespace model { namespace run { -public: - /** Define integral read number type */ - typedef size_t number_t; -public: - /** Constructor - * - * @param number read number identifier - * @param first_cycle index of first cycle - * @param last_cycle index of last cycle - * @param is_index true if the read is an index read - */ - read_info(const number_t number=0, - const cycle_t first_cycle=0, - const cycle_t last_cycle=0, - const bool is_index=false) : cycle_range(first_cycle, last_cycle), m_number(number), m_is_index(is_index) - { - } -public: - /** @defgroup read_info Read Information - * - * Information describing the read - * - * @ingroup run_info - * @ref illumina::interop::model::run::read_info "See full class description" - * @{ - */ - /** Get the number to identify the read - * - * @return number to identify the read - */ - number_t number()const - { - return m_number; - } - /** Check if read is index + /** Defines a range over cycles * - * @return true if read is index */ - bool is_index()const + class read_info : public cycle_range { - return m_is_index; - } - /** Get the total number of cycles in the read - * - * @return total number of cycles in read - */ - cycle_t total_cycles()const - { - return (m_last_cycle >= m_first_cycle) ? m_last_cycle-m_first_cycle+1 : 0; - } - /** Get the usable number of cycles in the read - * - * The last cycle is not fully corrected so it is not usable - * - * @return number of usable cycles in read - */ - cycle_t useable_cycles()const - { - return (m_last_cycle >= m_first_cycle) ? m_last_cycle-m_first_cycle : 0; - } - /** @} */ + public: + /** Define integral read number type */ + typedef size_t number_t; + public: + /** Constructor + * + * @param number read number identifier + * @param first_cycle index of first cycle + * @param last_cycle index of last cycle + * @param is_index true if the read is an index read + */ + read_info(const number_t number = 0, + const cycle_t first_cycle = 0, + const cycle_t last_cycle = 0, + const bool is_index = false) : cycle_range(first_cycle, last_cycle), m_number(number), + m_is_index(is_index) + { + } + + public: + /** @defgroup read_info Read Information + * + * Information describing the read + * + * @ingroup run_info + * @ref illumina::interop::model::run::read_info "See full class description" + * @{ + */ + /** Get the number to identify the read + * + * @return number to identify the read + */ + number_t number() const + { + return m_number; + } + + /** Check if read is index + * + * @return true if read is index + */ + bool is_index() const + { + return m_is_index; + } + + /** Get the total number of cycles in the read + * + * @return total number of cycles in read + */ + cycle_t total_cycles() const + { + return (m_last_cycle >= m_first_cycle) ? m_last_cycle - m_first_cycle + 1 : 0; + } + + /** Get the usable number of cycles in the read + * + * The last cycle is not fully corrected so it is not usable + * + * @return number of usable cycles in read + */ + cycle_t useable_cycles() const + { + return (m_last_cycle >= m_first_cycle) ? m_last_cycle - m_first_cycle : 0; + } + /** @} */ + + private: + size_t m_number; + bool m_is_index; + + friend class info; -private: - size_t m_number; - bool m_is_index; - friend class info; - template - friend struct io::generic_layout; -}; + template + friend + struct io::generic_layout; + }; -} -} -} -} +}}}} diff --git a/interop/model/run_metrics.h b/interop/model/run_metrics.h index e61a8cfce..2ae6a94a8 100644 --- a/interop/model/run_metrics.h +++ b/interop/model/run_metrics.h @@ -10,6 +10,11 @@ #include "interop/util/exception.h" #include "interop/util/object_list.h" #include "interop/model/metric_base/metric_set.h" +#include "interop/io/stream_exceptions.h" +#include "interop/model/run/info.h" +#include "interop/model/run/parameters.h" + +//Metrics #include "interop/model/metrics/corrected_intensity_metric.h" #include "interop/model/metrics/error_metric.h" #include "interop/model/metrics/extraction_metric.h" @@ -19,17 +24,10 @@ #include "interop/model/metrics/q_by_lane_metric.h" #include "interop/model/metrics/q_collapsed_metric.h" #include "interop/model/metrics/tile_metric.h" -#include "interop/io/metric_file_stream.h" -#include "interop/model/run/info.h" -#include "interop/model/run/parameters.h" -#include "interop/logic/metric/q_metric.h" -#include "interop/logic/metric/tile_metric.h" -#include "interop/logic/utils/channel.h" namespace illumina { namespace interop { namespace model { namespace metrics { - /** Collection of all metric sets for a run * * @ingroup run_metrics @@ -65,6 +63,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics typedef object_list metric_list_t; public: + /** Define an id type */ + typedef metric_base::base_metric::id_t id_t; /** Define corrected intensity metric set */ typedef metric_base::metric_set corrected_intensity_metric_set_t; /** Define error metric set */ @@ -83,6 +83,10 @@ namespace illumina { namespace interop { namespace model { namespace metrics typedef metric_base::metric_set q_collapsed_metric_set_t; /** Define by lane q-metric set */ typedef metric_base::metric_set q_by_lane_metric_set_t; + /** Define a map of ids to a base metric */ + typedef std::map tile_metric_map_t; + /** Define a map of ids to a base cycle metric */ + typedef std::map cycle_metric_map_t; public: @@ -111,7 +115,6 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @ref illumina::interop::model::metrics::run_metrics "See full class description" * @{ */ - /** Read binary metrics and XML files from the run folder * * @param run_folder run folder path @@ -126,12 +129,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics io::incomplete_file_exception, io::format_exception, model::index_out_of_bounds_exception, - model::invalid_tile_naming_method) - { - read_metrics(run_folder); - const size_t count = read_xml(run_folder); - finalize_after_load(count); - } + model::invalid_tile_naming_method, + model::invalid_run_info_exception); /** Read XML files: RunInfo.xml and possibly RunParameters.xml * @@ -142,11 +141,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics xml::bad_xml_format_exception, xml::empty_xml_format_exception, xml::missing_xml_element_exception, - xml::xml_parse_exception) - { - read_run_info(run_folder); - return read_run_parameters(run_folder); - } + xml::xml_parse_exception); /** Read RunInfo.xml * @@ -156,10 +151,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics xml::bad_xml_format_exception, xml::empty_xml_format_exception, xml::missing_xml_element_exception, - xml::xml_parse_exception) - { - m_run_info.read(run_folder); - } + xml::xml_parse_exception); /** Read RunParameters.xml if necessary * @@ -170,28 +162,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics xml::bad_xml_format_exception, xml::empty_xml_format_exception, xml::missing_xml_element_exception, - xml::xml_parse_exception) - { - const size_t count = logic::metric::count_legacy_q_score_bins(get_set()); - if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count)) - { - - try - { - m_run_parameters.read(run_folder); - } - catch (const xml::xml_file_not_found_exception &) - { - if (m_run_info.channels().empty()) - INTEROP_THROW(io::file_not_found_exception, - "RunParameters.xml required for legacy run folders with missing channel names"); - else - INTEROP_THROW(io::file_not_found_exception, - "RunParameters.xml required for legacy run folders and is missing"); - } - } - return count; - } + xml::xml_parse_exception); /** Finalize the metric sets after loading from disk * @@ -199,79 +170,26 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ void finalize_after_load(size_t count = std::numeric_limits::max()) throw(io::format_exception, model::invalid_tile_naming_method, - model::index_out_of_bounds_exception) - { - if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) - { - m_run_info.set_naming_method( - logic::metric::tile_naming_method_from_metric(get_set())); - if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) - m_run_info.set_naming_method( - logic::metric::tile_naming_method_from_metric(get_set())); - if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) - m_run_info.set_naming_method( - logic::metric::tile_naming_method_from_metric(get_set())); - } - if (count == std::numeric_limits::max()) - { - if(get_set().size() > 0) - count = logic::metric::count_legacy_q_score_bins(get_set()); - else if(get_set().size()) - count = logic::metric::count_legacy_q_score_bins(get_set()); - } - logic::metric::populate_legacy_q_score_bins(get_set().bins(), m_run_parameters.instrument_type(), - count); - if (get_set().size() > 0 && get_set().size() == 0) - logic::metric::create_collapse_q_metrics(get_set(), get_set()); - if (get_set().size() > 0 && get_set().size() == 0) - logic::metric::create_q_metrics_by_lane(get_set(), get_set()); - logic::metric::populate_cumulative_distribution(get_set()); - logic::metric::populate_cumulative_distribution(get_set()); - logic::metric::populate_cumulative_distribution(get_set()); - INTEROP_ASSERT(get_set().size() == 0 || get_set().size() == get_set().size()); - if (m_run_info.channels().empty()) - { - legacy_channel_update(m_run_parameters.instrument_type()); - if (m_run_info.channels().empty()) - INTEROP_THROW(io::format_exception, - "Channel names are missing from the RunInfo.xml, and RunParameters.xml does not contain sufficient information on the instrument run."); - } - if (!empty() && run_info().flowcell().naming_method() == constants::UnknownTileNamingMethod) - INTEROP_THROW(model::invalid_tile_naming_method, "Unknown tile naming method - update your RunInfo.xml"); - extraction_metric_set_t& extraction_metrics = get_set(); - // Trim excess channel data for imaging table - for(extraction_metric_set_t::iterator it = extraction_metrics.begin(); it != extraction_metrics.end();++it) - it->trim(run_info().channels().size()); - } + model::index_out_of_bounds_exception, + model::invalid_run_info_exception); /** Test if all metrics are empty * * @return true if all metrics are empty */ - bool empty() const - { - is_metric_empty func; - m_metrics.apply(func); - return func.empty(); - } + bool empty() const; /** Update channels for legacy runs * * @param type instrument type */ - void legacy_channel_update(const constants::instrument_type type) - { - m_run_info.channels(logic::utils::update_channel_from_instrument_type(type)); - } + void legacy_channel_update(const constants::instrument_type type); /** Set the tile naming method * * @param naming_method tile naming method */ - void set_naming_method(const constants::tile_naming_method naming_method) - { - m_run_info.set_naming_method(naming_method); - } + void set_naming_method(const constants::tile_naming_method naming_method); public: /** Get the set of corrected intensity metrics @@ -515,6 +433,13 @@ namespace illumina { namespace interop { namespace model { namespace metrics } public: + /** Check if the InterOp file for each metric set exists + * + * This will set the `metric_set::data_source_exists` flag. + * + * @param run_folder run folder path + */ + void check_for_data_sources(const std::string &run_folder); /** Read binary metrics from the run folder * * This function ignores: @@ -527,10 +452,20 @@ namespace illumina { namespace interop { namespace model { namespace metrics void read_metrics(const std::string &run_folder) throw( io::file_not_found_exception, io::bad_format_exception, - io::incomplete_file_exception) - { - m_metrics.apply(read_func(run_folder)); - } + io::incomplete_file_exception); + /** Write binary metrics to the run folder + * + * @param run_folder run folder path + */ + void write_metrics(const std::string &run_folder)const throw( + io::file_not_found_exception, + io::bad_format_exception); + + /** Validate whether the RunInfo.xml matches the InterOp files + * + * @throws invalid_run_info_exception + */ + void validate() throw(invalid_run_info_exception); /** Read binary metrics and XML files from the run folder * @@ -541,22 +476,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics { metrics_callback(func); func(m_run_info); - const size_t count = logic::metric::count_legacy_q_score_bins(get_set()); - if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count)) - { - func(m_run_parameters); - } - if (m_run_info.channels().empty()) - { - m_run_info.channels( - logic::utils::update_channel_from_instrument_type(m_run_parameters.instrument_type())); - if (m_run_info.channels().empty()) - INTEROP_THROW(io::format_exception, - "Channel names are missing from the RunInfo.xml, and RunParameters.xml does not contain sufficient information on the instrument run."); - - } - logic::metric::populate_legacy_q_score_bins(get_set().bins(), m_run_parameters.instrument_type(), - count); + finalize_after_load(); } /** Read binary metrics from the run folder @@ -568,63 +488,43 @@ namespace illumina { namespace interop { namespace model { namespace metrics { m_metrics.apply(func); } + /** Check if the metric group is empty + * + * @param group_name prefix of interop group metric + * @return true if metric is empty + */ + bool is_group_empty(const std::string& group_name) const; + /** Check if the metric group is empty + * + * @param group_id prefix of interop group metric id + * @return true if metric is empty + */ + bool is_group_empty(const constants::metric_group group_id) const; + + /** Populate a map of valid tiles + * + * @param map mapping between tile has and base_metric + */ + void populate_id_map(tile_metric_map_t &map) const; + + /** Populate a map of valid tiles and cycles + * + * @param map mapping between tile has and base_metric + */ + void populate_id_map(cycle_metric_map_t &map) const; + /** Sort the metrics by id + */ + void sort(); + + /** Clear all the metrics + */ + void clear(); private: metric_list_t m_metrics; run::info m_run_info; run::parameters m_run_parameters; - private: - struct is_metric_empty - { - is_metric_empty() : m_empty(true) - { } - - template - void operator()(const MetricSet &metrics) - { - if (metrics.size() > 0) m_empty = false; - } - - bool empty() const - { - return m_empty; - } - - bool m_empty; - }; - - struct read_func - { - read_func(const std::string &f) : m_run_folder(f) - { } - - template - int operator()(MetricSet &metrics) const - { - try - { - io::read_interop(m_run_folder, metrics); - } - catch (const io::file_not_found_exception &) - { - try - { - io::read_interop(m_run_folder, metrics, - false /** Search for XMetrics.bin not XMetricsOut.bin */); - } - catch (const io::file_not_found_exception &) - { return 1; } - catch (const io::incomplete_file_exception &) - { return 2; } - } - catch (const io::incomplete_file_exception &) - { return 2; } - return 0; - } - - std::string m_run_folder; - }; }; diff --git a/interop/model/summary/index_count_summary.h b/interop/model/summary/index_count_summary.h index 351a4731b..f9d711771 100644 --- a/interop/model/summary/index_count_summary.h +++ b/interop/model/summary/index_count_summary.h @@ -7,7 +7,7 @@ */ #pragma once #include -#include +#include "interop/util/math.h" namespace illumina { namespace interop { namespace model { namespace summary { diff --git a/interop/model/summary/index_lane_summary.h b/interop/model/summary/index_lane_summary.h index 2b778e1c2..9c070d871 100644 --- a/interop/model/summary/index_lane_summary.h +++ b/interop/model/summary/index_lane_summary.h @@ -7,9 +7,8 @@ */ #pragma once -#include - #include +#include "interop/util/cstdint.h" #include "interop/util/exception.h" #include "interop/model/summary/index_count_summary.h" diff --git a/interop/model/summary/lane_summary.h b/interop/model/summary/lane_summary.h index 62cffab91..90f535a7e 100644 --- a/interop/model/summary/lane_summary.h +++ b/interop/model/summary/lane_summary.h @@ -16,12 +16,12 @@ namespace illumina { namespace interop { namespace model { namespace summary { -/** Summary statistics compiled by lane - * - * @note The reported standard deviation is always between tiles, not cycles. For metrics like error metrics, - * the mean over all cycles is used for each tile, before calculating the standard deviation. - * - */ + /** Summary statistics compiled by lane + * + * @note The reported standard deviation is always between tiles, not cycles. For metrics like error metrics, + * the mean over all cycles is used for each tile, before calculating the standard deviation. + * + */ class lane_summary { public: @@ -309,118 +309,120 @@ namespace illumina { namespace interop { namespace model { namespace summary { m_reads_pf = val; } - /** Get statistics summarizing the density of tiles in the lane + /** Set statistics summarizing the density of tiles in the lane * - * @return statistics summarizing the density of tiles in the lane + * @param stat statistics summarizing the density of tiles in the lane */ - metric_stat_t &density() + void density(const metric_stat_t& stat) { - return m_density; + m_density = stat; } - /** Get statistics summarizing the passing filter density of tiles in the lane + /** Set statistics summarizing the passing filter density of tiles in the lane * - * @return statistics summarizing the passing filter density of tiles in the lane + * @param stat statistics summarizing the passing filter density of tiles in the lane */ - metric_stat_t &density_pf() + void density_pf(const metric_stat_t& stat) { - return m_density_pf; + m_density_pf = stat; } - /** Get statistics summarizing the cluster count of tiles in the lane + /** Set statistics summarizing the cluster count of tiles in the lane * - * @return statistics summarizing the cluster count of tiles in the lane + * @param stat statistics summarizing the cluster count of tiles in the lane */ - metric_stat_t &cluster_count() + void cluster_count(const metric_stat_t& stat) { - return m_cluster_count; + m_cluster_count = stat; } - /** Get statistics summarizing the passing filter cluster count of tiles in the lane + /** Set statistics summarizing the passing filter cluster count of tiles in the lane * - * @return statistics summarizing the passing filter cluster count of tiles in the lane + * @param stat statistics summarizing the passing filter cluster count of tiles in the lane */ - metric_stat_t &cluster_count_pf() + void cluster_count_pf(const metric_stat_t& stat) { - return m_cluster_count_pf; + m_cluster_count_pf = stat; } - /** Get statistics summarizing the percent of clusters passing filter of tiles in the lane + /** Set statistics summarizing the percent of clusters passing filter of tiles in the lane * - * @return statistics summarizing the percent of clusters passing filter of tiles in the lane + * @param stat statistics summarizing the percent of clusters passing filter of tiles in the lane */ - metric_stat_t &percent_pf() + void percent_pf(const metric_stat_t& stat) { - return m_percent_pf; + m_percent_pf = stat; } - /** Get statistics summarizing the phasing of tiles in the lane + /** Set statistics summarizing the phasing of tiles in the lane * - * @return statistics summarizing the phasing of tiles in the lane + * @param stat statistics summarizing the phasing of tiles in the lane */ - metric_stat_t &phasing() + void phasing(const metric_stat_t& stat) { - return m_phasing; + m_phasing = stat; } - /** Get statistics summarizing the prephasing of tiles in the lane + /** Set statistics summarizing the prephasing of tiles in the lane * - * @return statistics summarizing the prephasing of tiles in the lane + * @param stat statistics summarizing the prephasing of tiles in the lane */ - metric_stat_t &prephasing() + void prephasing(const metric_stat_t& stat) { - return m_prephasing; + m_prephasing = stat; } - /** Get statistics summarizing the PhiX percent aligned of tiles in the lane + /** Set statistics summarizing the PhiX percent aligned of tiles in the lane * - * @return statistics summarizing the PhiX percent aligned of tiles in the lane + * @param stat statistics summarizing the PhiX percent aligned of tiles in the lane */ - metric_stat_t &percent_aligned() + void percent_aligned(const metric_stat_t& stat) { - return m_percent_aligned; + m_percent_aligned = stat; } - /** Get statistics summarizing the PhiX error rate of tiles in the lane + /** Set statistics summarizing the PhiX error rate of tiles in the lane * - * @return statistics summarizing the PhiX error rate of tiles in the lane + * @param stat statistics summarizing the PhiX error rate of tiles in the lane */ - metric_stat_t &error_rate() + void error_rate(const metric_stat_t& stat) { - return m_error_rate; + m_error_rate = stat; } - /** Get statistics summarizing the PhiX error rate over the first 35 cycles of tiles in the lane + /** Set statistics summarizing the PhiX error rate over the first 35 cycles of tiles in the lane * - * @return statistics summarizing the PhiX error rate over the first 35 cycles of tiles in the lane + * @param stat statistics summarizing the PhiX error rate over the first 35 cycles of tiles in the lane */ - metric_stat_t &error_rate_35() + void error_rate_35(const metric_stat_t& stat) { - return m_error_rate_35; + m_error_rate_35 = stat; } - /** Get statistics summarizing the PhiX error rate over the first 50 cycles of tiles in the lane + /** Set statistics summarizing the PhiX error rate over the first 50 cycles of tiles in the lane * - * @return statistics summarizing the PhiX error rate over the first 50 cycles of tiles in the lane + * @param stat statistics summarizing the PhiX error rate over the first 50 cycles of tiles in the lane */ - metric_stat_t &error_rate_50() + void error_rate_50(const metric_stat_t& stat) { - return m_error_rate_50; + m_error_rate_50 = stat; } - /** Get statistics summarizing the PhiX error rate over the first 75 cycles of tiles in the lane + /** Set statistics summarizing the PhiX error rate over the first 75 cycles of tiles in the lane * - * @return statistics summarizing the PhiX error rate over the first 75 cycles of tiles in the lane + * @param stat statistics summarizing the PhiX error rate over the first 75 cycles of tiles in the lane */ - metric_stat_t &error_rate_75() + void error_rate_75(const metric_stat_t& stat) { - return m_error_rate_75; + m_error_rate_75 = stat; } - /** Get statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane + /** Set statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane * - * @return statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane + * @param stat statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane */ - metric_stat_t &error_rate_100() + void error_rate_100(const metric_stat_t& stat) { - return m_error_rate_100; + m_error_rate_100 = stat; } - /** Get statistics summarizing the first cycle intensity of tiles in the lane + /** Set statistics summarizing the first cycle intensity of tiles in the lane * - * @return statistics summarizing the first cycle intensity of tiles in the lane + * @param stat statistics summarizing the first cycle intensity of tiles in the lane */ - metric_stat_t &first_cycle_intensity() + void first_cycle_intensity(const metric_stat_t & stat) { - return m_first_cycle_intensity; + m_first_cycle_intensity=stat; } + + /** Get statistics summarizing the cycle of each RTA state of tiles in the lane * * @return statistics summarizing the cycle of each RTA state of tiles in the lane @@ -430,6 +432,17 @@ namespace illumina { namespace interop { namespace model { namespace summary return m_cycle_state; } + + /** Set statistics summarizing the cycle of each RTA state of tiles in the lane + * + * @param state statistics summarizing the cycle of each RTA state of tiles in the lane + */ + /*void cycle_state(cycle_state_summary& state) + { + m_cycle_state = state; + }*/ + + private: size_t m_lane; size_t m_tile_count; diff --git a/interop/model/table/column_header.h b/interop/model/table/column_header.h deleted file mode 100644 index 49c9208d5..000000000 --- a/interop/model/table/column_header.h +++ /dev/null @@ -1,105 +0,0 @@ -/** Model for the imaging table - * - * @file - * @date 6/27/16 - * @version 1.0 - * @copyright GNU Public License. - */ -#pragma once -#include -#include - -namespace illumina { namespace interop { namespace model { namespace table { - - - /** Column header for the imaging table - */ - struct column_header - { - /** Constructor */ - column_header(){} - /** Constructor - * - * @param title tile of the column - */ - column_header(const std::string& title) : m_subtitle(title) {} - /** Constructor - * - * @param title tile of the column - * @param subtitle subtile of the column - */ - column_header(const std::string& title, const std::string& subtitle) : m_title(title), m_subtitle(subtitle){} - - public: - /** Get the group title - * - * @return group title - */ - const std::string& title()const - { - return m_title; - } - /** Get the subtitle - * - * @return subtitle - */ - const std::string& subtitle()const - { - return m_subtitle; - } - /** Get the identifer title - * - * @return identifier title - */ - const std::string& id()const - { - return m_title != "" ? m_title : m_subtitle; - } - /** Compare the column header to another - * - * @param other other column header - * @return true if both titles are the same - */ - bool operator==(const column_header& other)const - { - return m_title == other.m_title && m_subtitle == other.m_subtitle; - } - /** Read a column header from an input stream - * - * @param in input stream - * @param header column header - * @return input stream - */ - friend std::istream& operator>>(std::istream& in, column_header& header) - { - std::getline(in, header.m_subtitle, ','); - std::string::size_type n; - if((n=header.m_subtitle.find("|")) != std::string::npos) - { - header.m_title = header.m_subtitle.substr(0, n); - header.m_subtitle = header.m_subtitle.substr(n+1); - } - return in; - } - - /** Write a column header to an output stream - * - * @param out output stream - * @param header column header - * @return output stream - */ - friend std::ostream& operator<<(std::ostream& out, const column_header& header) - { - if(header.m_title != "") out << header.m_title << "|"; - out << header.m_subtitle; - return out; - } - - private: - /** Title of a group of columns */ - std::string m_title; - /** Title of the column */ - std::string m_subtitle; - }; - -}}}} diff --git a/interop/model/table/imaging_column.h b/interop/model/table/imaging_column.h new file mode 100644 index 000000000..448b11178 --- /dev/null +++ b/interop/model/table/imaging_column.h @@ -0,0 +1,286 @@ +/** Model for the imaging table column + * + * @file + * @date 7/20/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include +#include +#include "interop/util/assert.h" +#include "interop/util/length_of.h" +#include "interop/util/string.h" +#include "interop/constants/enums.h" +#include "interop/logic/utils/enums.h" + + +/** Mapping of data for the imaging table + * + * Column 1: + * - Name of the field in the table_entry class + * - Converted to column header in imaging table + * - Suffixed with 'Column' when used as an enum, e.g. Lane -> LaneColumn + * Column 2: + * - metric that holds the column data + * Column 3: + * - method in metric that holds the column data + * Column 4: + * - Parameter taken by metric method + * e.g. percent_aligned requires a read number, which can be taken from table_entry + * e.g. percent_over_qscore needs to know the index of the bin to use + * e.g. surface needs to know the tile naming convention to properly parse the name + * Column 5: + * - Data type for the field in table_entry + * Column 6: + * - Flag for properties for this field + * Column 7: + * - Number of decimal places to round to + * + * The metrics marked "dummy" are populated not in the update function, but in the constructor. + * + * @note This macro requires the macro INTEROP_TUPLE7 to be defined before use + * @see illumina::interop::model::table::table_entry + * @see illumina::interop::logic::table::imaging_table_column_names + */ +#define INTEROP_IMAGING_COLUMN_TYPES \ + INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType, 0)\ + INTEROP_TUPLE7(Tile, metric_base::base_metric, tile, Void, UInt, IdType, 0)\ + INTEROP_TUPLE7(Cycle, metric_base::base_cycle_metric, cycle/*dummy*/, Void, UInt, IdType, 0)\ + INTEROP_TUPLE7(Read, metric_base::base_read_metric, read/*dummy*/, Void, UInt, IdType, 0)\ + INTEROP_TUPLE7(CycleWithinRead, metric_base::base_read_metric, id/*dummy*/, Void, UInt, IdType, 0)\ + INTEROP_TUPLE7(DensityKPermm2, metrics::tile_metric, cluster_density_k, Void, Float, ValueType, 1)\ + INTEROP_TUPLE7(DensityPfKPermm2, metrics::tile_metric, cluster_density_pf_k, Void, Float, ValueType, 1)\ + INTEROP_TUPLE7(ClusterCountK, metrics::tile_metric, cluster_count_k, Void, Float, ValueType, 1)\ + INTEROP_TUPLE7(ClusterCountPfK, metrics::tile_metric, cluster_count_pf_k, Void, Float, ValueType, 1)\ + INTEROP_TUPLE7(PercentPassFilter, metrics::tile_metric, percent_pf, Void, Float, ValueType, 1)\ + INTEROP_TUPLE7(PercentAligned, metrics::tile_metric, percent_aligned_at, Read, Float, ValueType, 1)\ + INTEROP_TUPLE7(PercentPhasing, metrics::tile_metric, percent_phasing_at, Read, Float, ValueType, 3)\ + INTEROP_TUPLE7(PercentPrephasing, metrics::tile_metric, percent_prephasing_at, Read, Float, ValueType, 3)\ + INTEROP_TUPLE7(ErrorRate, metrics::error_metric, error_rate, Void, Float, ValueType, 3)\ + INTEROP_TUPLE7(PercentGreaterThanQ20, metrics::q_metric, percent_over_qscore, Q20, Float, ValueType, 2)\ + INTEROP_TUPLE7(PercentGreaterThanQ30, metrics::q_metric, percent_over_qscore, Q30, Float, ValueType, 2)\ + INTEROP_TUPLE7(P90, metrics::extraction_metric, max_intensity_values, Void, UShort, ChannelArray, 0)\ + INTEROP_TUPLE7(PercentNoCalls, metrics::corrected_intensity_metric,percent_nocall, Void, Float, ValueType, 1)\ + INTEROP_TUPLE7(PercentBase, metrics::corrected_intensity_metric,percent_bases, Void, Float, BaseArray, 1)\ + INTEROP_TUPLE7(Fwhm, metrics::extraction_metric, focus_scores, Void, Float, ChannelArray, 2) \ + INTEROP_TUPLE7(Corrected, metrics::corrected_intensity_metric,corrected_int_all_array, Void, UShort, BaseArray, 0)\ + INTEROP_TUPLE7(Called, metrics::corrected_intensity_metric,corrected_int_called_array, Void, UShort, BaseArray, 0)\ + INTEROP_TUPLE7(SignalToNoise, metrics::corrected_intensity_metric,signal_to_noise, Void, Float, ValueType, 0)\ + INTEROP_TUPLE7(MinimumContrast, metrics::image_metric, min_contrast_array, Void, UShort, ChannelArray, 0)\ + INTEROP_TUPLE7(MaximumContrast, metrics::image_metric, max_contrast_array, Void, UShort, ChannelArray, 0)\ + INTEROP_TUPLE7(Time, metrics::extraction_metric, date_time_csharp, Void, DateTime, StructType, 0)\ + INTEROP_TUPLE7(Surface, metric_base::base_metric, surface, NamingConvention, UInt, IdType, 0)\ + INTEROP_TUPLE7(Swath, metric_base::base_metric, swath, NamingConvention, UInt, IdType, 0)\ + INTEROP_TUPLE7(Section, metric_base::base_metric, section, NamingConvention, UInt, IdType, 0)\ + INTEROP_TUPLE7(TileNumber, metric_base::base_metric, number, NamingConvention, UInt, IdType, 0) + +namespace illumina { namespace interop { namespace model { namespace table +{ +# define INTEROP_TUPLE7(Id, Ignored1, Ignored2, Ignored3, Ignored4, Ignored5, Ignored6) Id##Column, + /** Map column index to name. Note, the enum has a 'Column' suffix added to the name above*/ + enum column_id{ INTEROP_IMAGING_COLUMN_TYPES ImagingColumnCount, UnknownColumnId=INTEROP_UNKNOWN}; +# undef INTEROP_TUPLE7 + + /** Information describing a single column or a group of columns + */ + class imaging_column + { + public: + /** Define a vector of strings */ + typedef std::vector string_vector; + public: + /** Constructor */ + imaging_column() : m_id(UnknownColumnId), m_offset(0) {} + /** Constructor + * + * @param index column id + * @param offset offset of the data into the data array + */ + imaging_column(const size_t index, const size_t offset) : + m_id(static_cast(index)), + m_name(to_header(m_id)), + m_offset(offset) + { + INTEROP_ASSERT(index < ImagingColumnCount); + } + /** Constructor + * + * @param index column id + * @param offset offset of the data into the data array + * @param sub_columns sub column header names + */ + imaging_column(const size_t index, const size_t offset, const string_vector& sub_columns) : + m_id(static_cast(index)), + m_name(to_header(m_id)), + m_offset(offset), + m_subcolumn_names(sub_columns) + { + INTEROP_ASSERT(index < ImagingColumnCount); + } + + public: + /** Get the column id enum + * + * @return column id enum + */ + column_id id()const + { + return m_id; + } + /** Get the name of the column + * + * @return name of the column + */ + const std::string& name()const + { + return m_name; + } + /** Check if the column has sub column headers + * + * @return true if there are sub column headers + */ + bool has_children()const + { + return !m_subcolumn_names.empty(); + } + /** Get the offset into the data array + * + * @return offset into the data array + */ + size_t offset()const + { + return m_offset; + } + /** Get a vector of subcolumn headers + * + * @return vector of subcolumn headers + */ + const string_vector& subcolumns()const + { + return m_subcolumn_names; + } + /** Get the full name of the column and sub column + * + * @param sub_index index of subcolumn + * @return name of the column + */ + std::string full_name(const size_t sub_index)const throw(model::index_out_of_bounds_exception) + { + if (has_children()) + { + if(sub_index >= m_subcolumn_names.size()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Sub column index out of bounds"); + return m_name + "_" + m_subcolumn_names[sub_index]; + } + return m_name; + } + + public: + /** Update the offset of the column + * + * @param off new offset into data array + */ + void offset(const size_t off) + { + m_offset = off; + } + /** Update the offset of the column + * + * @param val new id + */ + void id(const column_id val) + { + m_id = val; + } + /** Update the offset of the column + * + * @param header header description for new id + */ + void parse_header_for_id(const std::string& header) + { + id(constants::parse(to_name(header))); + } + + public: + /** Get total number of subcolumns + * + * @return total number of subcolumns + */ + size_t size()const + { + return (has_children() ? m_subcolumn_names.size() : 1); + } + /** Get total number of columns in the data up to this column + * + * @return total number of columns in the data up to this column + */ + size_t column_count()const + { + return m_offset + size(); + } + + public: + /** Convert a column id enum to the string representation + * + * @param id column id enum + * @return header representation + */ + static std::string to_header(const column_id id) + { + return to_header(constants::to_string(id)); + } + /** Convert name to header + * + * @param name column name + * @return header representation + */ + static std::string to_header(const std::string& name) + { + std::string header = name; + util::replace(header, "Percent", "%"); + util::replace(header, "GreaterThan", ">="); + util::replace(header, "KPermm2", "(k/mm2)"); + util::replace(header, "K", " (k)"); + util::camel_to(header); + return header; + } + /** Convert header to name + * + * @param header column header name + * @return column name + */ + static std::string to_name(const std::string& header) + { + std::string name = header; + util::replace(name, "%", "Percent"); + util::replace(name, ">=", "GreaterThan"); + util::replace(name, "(k/mm2)", "KPermm2"); + util::replace(name, " (k)", "K"); + util::camel_from(name); + return name; + } + /** Convert header to name + * + * @param header column header + * @return column name + */ + static std::string to_name(const model::table::imaging_column& header) + { + return to_name(header.name()); + } + + private: + friend std::istream& operator>>(std::istream& in, imaging_column& column); + friend std::ostream& operator<<(std::ostream& out, const imaging_column& column); + + private: + column_id m_id; + std::string m_name; + size_t m_offset; + string_vector m_subcolumn_names; + }; + + +}}}} diff --git a/interop/model/table/imaging_table.h b/interop/model/table/imaging_table.h index 50439655e..2005243fb 100644 --- a/interop/model/table/imaging_table.h +++ b/interop/model/table/imaging_table.h @@ -6,131 +6,103 @@ * @copyright GNU Public License. */ #pragma once -#include -#include -#include "interop/model/table/column_header.h" -#include "interop/model/table/imaging_table_entry.h" +#include +#include "interop/util/math.h" +#include "interop/model/table/imaging_column.h" -namespace illumina { namespace interop { namespace model { namespace table { +namespace illumina { namespace interop { namespace model { namespace table +{ /** Describes an imaging table, row data, column headers and boolean filled */ class imaging_table { public: - /** Define a row vector */ - typedef std::vector< table_entry> row_vector_t; - /** Define a flag vector */ - typedef std::vector< bool > filled_vector_t; - /** Define a header vector */ - typedef std::vector< column_header > header_vector_t; + /** Define a column vector */ + typedef std::vector< imaging_column> column_vector_t; + /** Define a data vector */ + typedef std::vector< float > data_vector_t; + + private: + /** Define a data vector */ + typedef std::vector< size_t > offset_vector_t; + public: - /** Define a constant iterator to a row */ - typedef row_vector_t::const_iterator const_iterator; - /** Define an iterator to a row */ - typedef row_vector_t::iterator iterator; - /** Define a constant reference to a row */ - typedef row_vector_t::const_reference const_reference; + /** Constructor */ + imaging_table() : m_row_count(0), m_col_count(0){} public: /** Reserve space for the rows * - * @param n number of rows - */ - void reserve(const size_t n) - { - m_rows.reserve(n); + * @param rows number of rows + * @param cols column vector + * @param data table cell data + */ + void set_data(const size_t rows, column_vector_t& cols, data_vector_t& data) + { + if(cols.empty()) + { + clear(); + return; + } + m_columns.swap(cols); + m_data.swap(data); + m_row_count = rows; + m_col_count = m_columns.back().column_count(); + m_enum_to_index.assign(ImagingColumnCount, static_cast(ImagingColumnCount)); + for(size_t i=0;i=m_rows.size()) throw model::index_out_of_bounds_exception("Row out of bounds"); - return m_rows[n]; + if(static_cast(c) >= m_enum_to_index.size()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Invalid enum id for column"); + const size_t col = m_enum_to_index[static_cast(c)]; + if(col >= m_enum_to_index.size()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Invalid enum - column not filled"); + return at(r, col, subcol); } /** Get a reference to a row * - * @param n index of the row - * @return row + * @param r row row index + * @param c col column index + * @param subcol sub column offset + * @return cell value */ - const table_entry& at(const size_t n)const + float operator()(const size_t r, const size_t c, const size_t subcol=0)const { - if(n >=m_rows.size()) throw model::index_out_of_bounds_exception("Row out of bounds"); - return m_rows[n]; + return at(r, c, subcol); } - /** Add a row to the table + /** Get a reference to a row * - * @param val table row + * @param r row row index + * @param c col column index + * @param subcol sub column offset + * @return cell value */ - void push_back(const table_entry& val) + float at(const size_t r, const size_t c, const size_t subcol=0)const throw(model::index_out_of_bounds_exception) { - m_rows.push_back(val); + if(r >=m_row_count) INTEROP_THROW(model::index_out_of_bounds_exception, "Row out of bounds"); + if(c >=m_columns.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Column out of bounds"); + const size_t col = m_columns[c].offset()+subcol; + if(col >=m_col_count) INTEROP_THROW(model::index_out_of_bounds_exception, "Column data offset out of bounds"); + const size_t index = r*m_col_count+col; + INTEROP_ASSERT(index < m_data.size()); + return m_data[index]; } /** Get a vector of column headers * * @return column headers */ - const header_vector_t& headers()const - { - return m_columns_header; - } - /** Set a vector of column headers - * - * @param headers column headers - */ - void headers(const header_vector_t& headers) - { - m_columns_header = headers; - } - /** Get a vector of filled entries - * - * @return bool vector - */ - const filled_vector_t& filled_columns()const - { - return m_columns_filled; - } - /** Set a vector of filled entries - * - * @param vec filled vector - */ - void filled_columns(const filled_vector_t& vec) - { - m_columns_filled = vec; - } - /** Get iterator to first row - * - * @return iterator to first row - */ - const_iterator begin()const - { - return m_rows.begin(); - } - /** Get iterator to end of row collection - * - * @return iterator - */ - const_iterator end()const - { - return m_rows.end(); - } - /** Get iterator to first row - * - * @return iterator to first row - */ - iterator begin() - { - return m_rows.begin(); - } - /** Get iterator to end of row collection - * - * @return iterator - */ - iterator end() + const column_vector_t& columns()const { - return m_rows.end(); + return m_columns; } /** Test if the table is empty * @@ -138,33 +110,44 @@ namespace illumina { namespace interop { namespace model { namespace table { */ bool empty()const { - return m_rows.empty(); + return m_data.empty(); } - /** Resize the number of rows - * - * @param n number of rows + /** Clear the contents of the table */ - void resize(const size_t n) + void clear() { - m_rows.resize(n); + m_data.clear(); + m_columns.clear(); + m_col_count = 0; + m_row_count = 0; } - /** Clear the contents of the table + /** Get the current column description (which may have subcolumns) + * + * @param col_index index of column description + * @return column description */ - void clear() + const imaging_column& column_at(const size_t col_index)throw(model::index_out_of_bounds_exception) { - m_rows.clear(); - m_columns_filled.clear(); - m_columns_header.clear(); + if( col_index >= m_columns.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Column index of out bounds"); + return m_columns[col_index]; } public: - /** Number of columns in the table + /** Number of columns * * @return column count */ size_t column_count()const { - return m_columns_header.size(); + return m_columns.size(); + } + /** Number of columns including subcolumns in the table + * + * @return column count + */ + size_t total_column_count()const + { + return m_col_count; } /** Number of rows in the table * @@ -172,15 +155,56 @@ namespace illumina { namespace interop { namespace model { namespace table { */ size_t row_count()const { - return m_rows.size(); + return m_row_count; } + private: friend std::istream& operator>>(std::istream& in, imaging_table& table); friend std::ostream& operator<<(std::ostream& out, const imaging_table& table); private: - row_vector_t m_rows; - filled_vector_t m_columns_filled; - header_vector_t m_columns_header; + data_vector_t m_data; + column_vector_t m_columns; + offset_vector_t m_enum_to_index; + size_t m_row_count; + size_t m_col_count; + }; + /** Compare two row indicies to determine which id in the table is less than another row + */ + struct imaging_table_id_less + { + /** Constructor + * + * @param table reference to an imaging table + */ + imaging_table_id_less(const imaging_table& table) : m_table(table){} + + /** Test if one row is less than another in terms of the tile id/cycle ordering + * + * @param lhs_row index to one row + * @param rhs_row index to another row + * @return true if the id in the first row is less than the second + */ + bool operator()(const size_t lhs_row, const size_t rhs_row)const + { + const size_t lhs_lane = static_cast(m_table(lhs_row, LaneColumn)); + const size_t rhs_lane = static_cast(m_table(rhs_row, LaneColumn)); + if(lhs_lane == rhs_lane) + { + const size_t lhs_tile = static_cast(m_table(lhs_row, TileColumn)); + const size_t rhs_tile = static_cast(m_table(rhs_row, TileColumn)); + if(lhs_tile == rhs_tile) + { + const size_t lhs_cycle = static_cast(m_table(lhs_row, CycleColumn)); + const size_t rhs_cycle = static_cast(m_table(rhs_row, CycleColumn)); + return lhs_cycle < rhs_cycle; + } + return lhs_tile < rhs_tile; + } + return lhs_lane < rhs_lane; + } + + private: + const imaging_table& m_table; }; }}}} diff --git a/interop/model/table/imaging_table_entry.h b/interop/model/table/imaging_table_entry.h deleted file mode 100644 index 72b208851..000000000 --- a/interop/model/table/imaging_table_entry.h +++ /dev/null @@ -1,454 +0,0 @@ -/** Model for a row in the imaging table - * - * @file - * @date 5/12/16 - * @version 1.0 - * @copyright GNU Public License. - */ -#pragma once -#include -#include "interop/model/metrics/error_metric.h" -#include "interop/model/metrics/extraction_metric.h" -#include "interop/model/metrics/corrected_intensity_metric.h" -#include "interop/model/metrics/tile_metric.h" -#include "interop/model/table/table_util.h" - -/** Mapping of data for the imaging table - * - * Column 1: - * - Name of the field in the table_entry class - * - Converted to column header in imaging table - * - Suffixed with 'Column' when used as an enum, e.g. Lane -> LaneColumn - * Column 2: - * - metric that holds the column data - * Column 3: - * - method in metric that holds the column data - * Column 4: - * - Parameter taken by metric method - * e.g. percent_aligned requires a read number, which can be taken from table_entry - * e.g. percent_over_qscore needs to know the index of the bin to use - * e.g. surface needs to know the tile naming convention to properly parse the name - * Column 5: - * - Data type for the field in table_entry - * Column 6: - * - Flag for properties for this field - * Column 7: - * - Number of decimal places to round to - * - * The metrics marked "dummy" are populated not in the update function, but in the constructor. - * - * @note This macro requires the macro INTEROP_TUPLE7 to be defined before use - * @see illumina::interop::model::table::table_entry - * @see illumina::interop::logic::table::imaging_table_column_names - */ -#define INTEROP_IMAGING_COLUMN_TYPES \ - INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType, 0)\ - INTEROP_TUPLE7(Tile, metric_base::base_metric, tile, Void, UInt, IdType, 0)\ - INTEROP_TUPLE7(Cycle, metric_base::base_cycle_metric, cycle/*dummy*/, Void, UInt, IdType, 0)\ - INTEROP_TUPLE7(Read, metric_base::base_read_metric, read/*dummy*/, Void, UInt, IdType, 0)\ - INTEROP_TUPLE7(CycleWithinRead, metric_base::base_read_metric, id/*dummy*/, Void, UInt, IdType, 0)\ - INTEROP_TUPLE7(DensityKPermm2, metrics::tile_metric, cluster_density_k, Void, Float, ValueType, 1)\ - INTEROP_TUPLE7(DensityPfKPermm2, metrics::tile_metric, cluster_density_pf_k, Void, Float, ValueType, 1)\ - INTEROP_TUPLE7(ClusterCountK, metrics::tile_metric, cluster_count_k, Void, Float, ValueType, 1)\ - INTEROP_TUPLE7(ClusterCountPfK, metrics::tile_metric, cluster_count_pf_k, Void, Float, ValueType, 1)\ - INTEROP_TUPLE7(PercentPassFilter, metrics::tile_metric, percent_pf, Void, Float, ValueType, 1)\ - INTEROP_TUPLE7(PercentAligned, metrics::tile_metric, percent_aligned_at, Read, Float, ValueType, 1)\ - INTEROP_TUPLE7(PercentPhasing, metrics::tile_metric, percent_phasing_at, Read, Float, ValueType, 3)\ - INTEROP_TUPLE7(PercentPrephasing, metrics::tile_metric, percent_prephasing_at, Read, Float, ValueType, 3)\ - INTEROP_TUPLE7(ErrorRate, metrics::error_metric, error_rate, Void, Float, ValueType, 3)\ - INTEROP_TUPLE7(PercentGreaterThanQ20, metrics::q_metric, percent_over_qscore, Q20, Float, ValueType, 2)\ - INTEROP_TUPLE7(PercentGreaterThanQ30, metrics::q_metric, percent_over_qscore, Q30, Float, ValueType, 2)\ - INTEROP_TUPLE7(P90, metrics::extraction_metric, max_intensity_values, Void, UShort, ChannelArray, 0)\ - INTEROP_TUPLE7(PercentNoCalls, metrics::corrected_intensity_metric,percent_nocall, Void, Float, ValueType, 1)\ - INTEROP_TUPLE7(PercentBase, metrics::corrected_intensity_metric,percent_bases, Void, Float, BaseArray, 1)\ - INTEROP_TUPLE7(Fwhm, metrics::extraction_metric, focus_scores, Void, Float, ChannelArray, 2) \ - INTEROP_TUPLE7(Corrected, metrics::corrected_intensity_metric,corrected_int_all_array, Void, UShort, BaseArray, 0)\ - INTEROP_TUPLE7(Called, metrics::corrected_intensity_metric,corrected_int_called_array, Void, UShort, BaseArray, 0)\ - INTEROP_TUPLE7(SignalToNoise, metrics::corrected_intensity_metric,signal_to_noise, Void, Float, ValueType, 0)\ - INTEROP_TUPLE7(MinimumContrast, metrics::image_metric, min_contrast_array, Void, UShort, ChannelArray, 0)\ - INTEROP_TUPLE7(MaximumContrast, metrics::image_metric, max_contrast_array, Void, UShort, ChannelArray, 0)\ - INTEROP_TUPLE7(Time, metrics::extraction_metric, date_time_csharp, Void, DateTime, StructType, 0)\ - INTEROP_TUPLE7(Surface, metric_base::base_metric, surface, NamingConvention, UInt, IdType, 0)\ - INTEROP_TUPLE7(Swath, metric_base::base_metric, swath, NamingConvention, UInt, IdType, 0)\ - INTEROP_TUPLE7(Section, metric_base::base_metric, section, NamingConvention, UInt, IdType, 0)\ - INTEROP_TUPLE7(TileNumber, metric_base::base_metric, number, NamingConvention, UInt, IdType, 0) - -namespace illumina { namespace interop { namespace model { namespace table { -# define INTEROP_TUPLE7(Id, Ignored1, Ignored2, Ignored3, Ignored4, Ignored5, Ignored6) Id##Column, - /** Map column index to name. Note, the enum has a 'Column' suffix added to the name above*/ - enum column_type{ INTEROP_IMAGING_COLUMN_TYPES ImagingColumnCount}; -# undef INTEROP_TUPLE7 - - - /** Single row for the imaging table - */ - struct table_entry - { - /** Defines the type for the Id columns */ - typedef ::uint32_t UIntIdType; - /** Defines the type for the metric value columns */ - typedef float FloatValueType; - /** Defines the type for the ushort array columns */ - //typedef metrics::extraction_metric::ushort_array_t ShortChannelArray; - /** Defines the type for the float array columns */ - typedef metrics::extraction_metric::float_array_t FloatChannelArray; - /** Defines the type for the ushort array columns */ - typedef metrics::extraction_metric::ushort_array_t UShortChannelArray; - /** Defines the type for the float array columns */ - typedef metrics::corrected_intensity_metric::float_array_t FloatBaseArray; - /** Defines the type for the float array columns */ - typedef metrics::corrected_intensity_metric::ushort_array_t UShortBaseArray; - /** Shortens tile_naming_method to naming_method */ - typedef constants::tile_naming_method naming_method; - /** Date time object type */ - typedef util::csharp_date_time DateTimeStructType; - /** Defines unsigned int */ - typedef metrics::q_metric::uint_t uint_t; - /** Define an offset vector for filling a table */ - typedef std::vector< std::vector< size_t > > table_fill_vector_t; - /** Place holder enum type */ - enum void_type { - /** Place holder for functions that take no parameters */ - Void - }; - /* Defines the initialization of the fields in this class. - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * table_entry() : Lane(), Q20(), Q30(), NamingConvention(0) {} - */ -# define INTEROP_TUPLE7(Id, Ignored1, Ignored2, Ignored3, Ignored4, Ignored5, Ignored6) Id(), - /** Constructor - */ - table_entry() : INTEROP_IMAGING_COLUMN_TYPES Q20(), Q30(), NamingConvention(constants::UnknownTileNamingMethod){} - /** Constructor - * - * @param metric metric record - * @param read read number - * @param cycle_in_read cycle number within the read - * @param q20_idx index of Q20 bin - * @param q30_idx index of Q30 bin - * @param naming_convention tile naming method - */ - template - table_entry(const Metric& metric, - const size_t read, - const size_t cycle_in_read, - const size_t q20_idx, - const size_t q30_idx, - const naming_method naming_convention) : - INTEROP_IMAGING_COLUMN_TYPES - Q20(static_cast(q20_idx)), - Q30(static_cast(q30_idx)), - NamingConvention(naming_convention) - { - Cycle = metric.cycle(); - Read = static_cast(read); - CycleWithinRead = static_cast(cycle_in_read); - update(metric); - } -# undef INTEROP_TUPLE7 // Reuse this for another conversion - - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * Create a field for the table_entry class - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * IdType Lane; - * - * @see INTEROP_IMAGING_COLUMN_TYPES - */ -# define INTEROP_TUPLE7(Id, Ignored1, Ignored2, Ignored3, Type, Group, Ignored4) Type##Group Id; - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - private: - // Additional parameters when getting the appropriate metric - uint_t Q20; // Used as a parameter to percent_over_qscore to get PercentGreaterThanQ20 - uint_t Q30; // Used as a parameter to percent_over_qscore to get PercentGreaterThanQ30 - naming_method NamingConvention; // Used as a parameter to swath, section, surface, tile - // Read (one of the column fields, is used to get percent_aligned, percent_phasing, percent_prephasing - public: - /** Update the table entry with the given metric - * - * @param metric source metric to populate the row of data - */ - template - void update(const Metric& metric) - { - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * Add a method call to update each field with the given `metric` - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * update_laneVoid(metric, q20_idx, q30_idx, naming_convention); - */ -# define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) update_##Method##Param(metric); - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - } - /** Update the boolean filled metric array for each column based on whether the metric set is not empty - * - * @param metrics source metric set to populate the row of data - * @param filled boolean vector of flags indicating when the column is filled with data - * @param force_true force the filled column to be set true - */ - template - static void set_filled(const MetricSet& metrics, std::vector& filled, const bool force_true=false) - { - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * Add a method call to set a boolean flag to true if the column is occupied by values - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * set_filled_laneVoid(metric, filled, force); - */ -# define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) set_filled_##Method##Param(metrics, filled, force_true); - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - } - /** Update the filled metric array for each column based on whether the value is valid - * - * @note While this will work for any metric, it is intended only for tile_metrics. Unlike other metrics, - * tile_metrics may have partial entries. - * - * @param metric source metric to populate the row of data - * @param filled boolean vector of flags indicating when the column is filled with data - */ - template - void set_filled_metric(const Metric& metric, std::vector& filled)const - { - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * Add a method call to set a boolean flag to true if the column is occupied by valid values - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * set_filled_metric_laneVoid(metric, filled); - */ -# define INTEROP_TUPLE7(Ignore1, Ignore2, Method, Param, Ignore4, Ignore5, Ignored6) set_filled_metric_##Method##Param(metric, filled); - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - } - /** Set all the id columns as filled - * - * @note Unlike the case of base_metric, nothing derives from metric_set, so this must be explicit - * @param filled boolean vector of flags indicating when the column is filled with data - */ - static void set_id_filled(std::vector& filled) - { - set_filled(metric_base::metric_set(), filled, true); - set_filled(metric_base::metric_set(), filled, true); - set_filled(metric_base::metric_set(), filled, true); - } - /** Copy all fields to a vector - * - * @param vec vector of values - * @param filled boolean vector indicating which columns to copy - */ - template - void copy_to_vector(std::vector& vec, const std::vector& filled)const - { - vec.clear(); -# define INTEROP_TUPLE7(Id, Ignore2, Ignore3, Ignore4, Ignore5, Ignore6, Ignored7) copy_to_vector_##Id(vec, filled); - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - } - /** Fill all fields from a vector - * - * @param vec vector of values - * @param offsets enum for each value in vec - */ - template - void copy_from_vector(const std::vector& vec, const table_fill_vector_t& offsets) - { -# define INTEROP_TUPLE7(Id, Ignore2, Ignore3, Ignore4, Ignore5, Ignore6, Ignored7) copy_from_vector_##Id(vec, offsets); - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - } - - private: - template - static void copy_from_vector_impl(const std::vector& vec, DateTimeStructType& val, const std::vector offsets) - { - INTEROP_ASSERT(offsets.size()==1); - val = static_cast< ::uint64_t >(vec[offsets[0]]); - } - template - static void copy_from_vector_impl(const std::vector& vec, T& val, const std::vector offsets) - { - INTEROP_ASSERT(offsets.size()==1); - val = T(vec[offsets[0]]); - } - template - static void copy_from_vector_impl(const std::vector& vec, - std::vector& val, - const std::vector offsets) - { - INTEROP_ASSERT(offsets.size()>0); - val.clear(); - val.reserve(offsets.size()); - for(size_t i=0;i::min()) continue; - val.push_back(static_cast(vec[offsets[i]])); - } - } - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * This macro creates a single function that copies elements from a vector to a field - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * void copy_to_vector_Lane(std::vector& vec) - * - */ -# define INTEROP_TUPLE7(Id, Metric, Method, Param, Type, Kind, Round) \ - template\ - void copy_from_vector_##Id(const std::vector& vec, const table_fill_vector_t& offsets)\ - {\ - INTEROP_ASSERT(table::Id##Column < offsets.size());\ - if(!offsets[table::Id##Column].empty()) copy_from_vector_impl(vec, Id, offsets[table::Id##Column]);\ - } - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - - private: - template - static void copy_to_vector_impl(std::vector& vec, const T val) - { - vec.push_back(static_cast(val)); - } - template - static void copy_to_vector_impl(std::vector& vec, const std::vector& val) - { - for(typename std::vector::const_iterator it = val.begin();it != val.end();++it) - vec.push_back(static_cast(*it)); - } - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * This macro creates a single function that copies the field to a vector - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * void copy_to_vector_Lane(std::vector& vec) - * - */ -# define INTEROP_TUPLE7(Id, Metric, Method, Param, Type, Kind, Round) \ - template\ - void copy_to_vector_##Id(std::vector& vec, const std::vector& filled)const\ - {\ - if(filled[table::Id##Column]) copy_to_vector_impl(vec, Id);\ - } - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * This macro creates two functions, one to populate a field/column with the data from the correspond metric - * and an empty method to ignore a group - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * void update_laneVoid(const model::metric_base::base_metric& metric, const size_t Q20, const size_t Q30, const naming_method NamingConvention) - * void update_laneVoid(const model::metric_base::empty_metric&) - * - * @note Param can be can field in this class, e.g. Read, or the function parameters Q20, Q30 or NamingConvention - */ -# define INTEROP_TUPLE7(Id, Metric, Method, Param, Type, Kind, Round) \ - void update_##Method##Param(const Metric& metric)\ - {\ - Id = static_cast(call_adapter(metric, Param, &Metric::Method));\ - roundto(Id, Round); \ - (void)Q20;(void)Q30;(void)NamingConvention;\ - }\ - void update_##Method##Param(const model::metric_base::empty_metric&){} - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * This macro creates two functions, one to set the filled array to true if the metric set is not emtpy - * and the other does nothing. - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * void set_filled_laneVoid(const metric_base::metric_set& metrics, std::vector& filled, const bool force_true) - * void set_filled_laneVoid(const model::metric_base::empty_header&, std::vector&, const bool) - * - */ -# define INTEROP_TUPLE7(Id, Metric, Method, Param, Ignored1, Ignored2, Ignored3) \ - static void set_filled_##Method##Param(const metric_base::metric_set& metrics, std::vector& filled, const bool force_true)\ - {\ - filled[table::Id##Column] = (force_true) ? true : metrics.size()>0;\ - }\ - static void set_filled_##Method##Param(const model::metric_base::empty_header&, std::vector&, const bool){} - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - /* For every entry in INTEROP_IMAGING_COLUMN_TYPES - * This macro creates two functions, one to mark a column as filled if there is at least one valid metric - * - * Example: - * INTEROP_TUPLE7(Lane, metric_base::base_metric, lane, Void, UInt, IdType) -> - * - * void set_filled_metric_laneVoid(const Metric& metric, std::vector& filled)const - * void set_filled_metric_laneVoid(const model::metric_base::empty_metric&, std::vector&)const - */ -# define INTEROP_TUPLE7(Id, Metric, Method, Param, Ignored1, Ignored2, Ignored3) \ - void set_filled_metric_##Method##Param(const Metric& metric, std::vector& filled)const\ - {\ - if(is_valid(call_adapter(metric, Param, &Metric::Method))) filled[table::Id##Column] = true;\ - }\ - void set_filled_metric_##Method##Param(const model::metric_base::empty_metric&, std::vector&)const{} - INTEROP_IMAGING_COLUMN_TYPES -# undef INTEROP_TUPLE7 // Reuse this for another conversion - private: - inline static void roundto(float& val, const size_t n) - { - if(n>0) - { - double mult=1.0; - for(size_t i=0;i(std::floor(val*mult+0.5)/mult); - } - } - inline static void roundto(FloatChannelArray& vals, const size_t n) - { - for(FloatChannelArray::iterator b = vals.begin(), e = vals.end();b != e;++b) roundto(*b, n); - } - template - inline static void roundto(T&, const size_t) { } - /** Test if a metric type is valid - * - * @param val floating point value - * @return true if not NaN - */ - static bool is_valid(const float val) - { - return !std::isnan(val); - } - /** Test if a metric type is valid - * - * @param val integral value - * @return true - */ - template - static bool is_valid(const T) - { - return true; - } - /** Test if a metric type is valid - * - * @param values vector of values - * @return true if not empty - */ - template - static bool is_valid(const std::vector& values) - { - return !values.empty(); - } - }; - -}}}} diff --git a/interop/model/table/table_util.h b/interop/model/table/table_util.h deleted file mode 100644 index 2705b8bef..000000000 --- a/interop/model/table/table_util.h +++ /dev/null @@ -1,44 +0,0 @@ -/** Utility functions for building a table - * - * @note These functions were move here as a work around for SWIG - * - * @file - * @date 6/13/16 - * @version 1.0 - * @copyright GNU Public License. - */ -#pragma once - - -namespace illumina { namespace interop { namespace model { namespace table { - - /** Function interface for method call with single parameter - * - * @note This is necessary because you cannot pass anything into an function that expects no arguments, not even void - * - * @param obj object corresponding to the method - * @param param1 first value to function - * @param func pointer to member function - * @return functor wrapper - */ - template - R call_adapter(const T& obj, P2 param1, R (T::*func )(P1)const) - { - return (obj.*func)(param1); - } - /** Function interface for method call with single dummy parameter - * - * @note This is necessary because you cannot pass anything into an function that expects no arguments, not even void - * - * @param obj object corresponding to the method - * @param func pointer to member function - * @return functor wrapper - */ - template - R call_adapter(const T& obj, P1, R (T::*func )()const) - { - return (obj.*func)(); - } - - -}}}} diff --git a/interop/util/constant_mapping.h b/interop/util/constant_mapping.h index abd189d09..4c9a2804f 100644 --- a/interop/util/constant_mapping.h +++ b/interop/util/constant_mapping.h @@ -1,11 +1,11 @@ /** Map a constant of one type to another type * - * This is designed to simplify mapping between enums, enums to strings or strings built using #define statements. + * This is designed to simplify mapping between enums, enums to strings or strings built using define statements. * * @file * @date 8/9/15 * @version 1.0 - * @opyright GNU Public License. + * @copyright GNU Public License. */ #pragma once diff --git a/interop/util/exception_specification.h b/interop/util/exception_specification.h deleted file mode 100644 index c7a4c4be9..000000000 --- a/interop/util/exception_specification.h +++ /dev/null @@ -1,20 +0,0 @@ -/** Exception specification - * - * The header provides exceptions for specific errors that may be raised while reading a binary InterOp file. - * - * @todo Expand and use this funcationality - * @file - * @date 4/6/16 - * @version 1.0 - * @copyright GNU Public License. - */ -#pragma once - -#ifndef _INTEROP_NO_EXCEPTION_SPECIFICATION -#define _INTEROP_HELPER(X) X -#define _INTEROP_THROWS(X) throw(X) -#define _INTEROP_THROWS2(X, Y) throw(X, Y) -#else -#define _INTEROP_THROWS(X) -#define _INTEROP_THROWS2(X, Y) -#endif diff --git a/interop/util/filesystem.h b/interop/util/filesystem.h index 55658cb92..9a1d48dd9 100644 --- a/interop/util/filesystem.h +++ b/interop/util/filesystem.h @@ -3,119 +3,50 @@ * This header provides facilities to manipulate files, directories and the paths that identify them. * * @file - * * @date 8/9/15 * @version 1.0 * @copyright GNU Public License. */ - -#ifdef WIN32 -/** Platform dependent path separator */ -#define INTEROP_OS_SEP '\\' -#else -/** Platform dependent path separator */ -#define INTEROP_OS_SEP '/' -#endif +#pragma once #include -#include -#include -#pragma once -namespace illumina { - namespace interop { - namespace io { - /** Combine two directories or a directory and a filename into a file path - * - * This function provides a platform independent way to generate a file path. It currently supports most - * operating systems include Mac OSX, Windows and Linux/Unix. - * - * @param path string representing a file path - * @param name string representing a file or directory name to append to the end of the path - * @return proper os-dependent file path - */ - inline std::string combine(const std::string& path, const std::string& name) { - if (path != "" && path[path.size() - 1] != INTEROP_OS_SEP && name != "" && name[0] != INTEROP_OS_SEP) { - return path + INTEROP_OS_SEP + name; - } - return path + name; - } - namespace detail { -#ifndef WIN32 - /** Helper functor to match path separator on Linux - */ - struct match_path_sep - { - /** Test if given character is Linux path separator - * - * @param ch character to test - */ - bool operator()(char ch)const - { - return ch == '/'; - } - }; -#else - /** Helper functor to match path separator on Windows - */ - struct match_path_sep - { - /** Test if given character is Windows path separator - * - * @param ch character to test - */ - bool operator()(char ch)const - { - return ch == '/' || ch == '\\'; - } - }; -#endif - } - /** Get the file name from a file path - * - * @param source full file path - * @return name of the file - */ - inline std::string basename(std::string const& source) - { - if(source.empty()) return ""; - if( detail::match_path_sep()(source[source.length()-1] ) ) - { - return std::string(std::find_if(source.rbegin()+1, source.rend(), detail::match_path_sep()).base(), source.end()-1); - } - return std::string(std::find_if(source.rbegin(), source.rend(), detail::match_path_sep()).base(), source.end()); - } - /** Get the directory name from a file path - * - * @param source full file path - * @return name of the directory - */ - inline std::string dirname(std::string source) - { - if (source.size() <= 1) //Make sure it's possible to check the last character. - { - return source; - } - detail::match_path_sep is_sep; - if (is_sep(*(source.rbegin() + 1))) //Remove trailing slash if it exists. - { - source = source.substr(0, source.size()-1); - // source.pop_back(); // C++11 - } - source.erase(std::find_if(source.rbegin(), source.rend(), is_sep).base(), source.end()); - return source; - } - /** Check if a file exists - * - * @param filename name of the file - * @return true if the file exists and is readable - */ - inline bool is_file_readable(const std::string& filename) - { - std::ifstream fin(filename.c_str()); - return fin.good(); - } - } - } -} +namespace illumina { namespace interop { namespace io +{ + /** Combine two directories or a directory and a filename into a file path + * + * This function provides a platform independent way to generate a file path. It currently supports most + * operating systems include Mac OSX, Windows and Linux/Unix. + * + * @param path string representing a file path + * @param name string representing a file or directory name to append to the end of the path + * @return proper os-dependent file path + */ + std::string combine(const std::string& path, const std::string& name); + /** Get the file name from a file path + * + * @param source full file path + * @return name of the file + */ + std::string basename(std::string const& source); + /** Get the directory name from a file path + * + * @param source full file path + * @return name of the directory + */ + std::string dirname(std::string source); + /** Check if a file exists + * + * @param filename name of the file + * @return true if the file exists and is readable + */ + bool is_file_readable(const std::string& filename); + /** Create a directory + * + * @param path path to new directory + * @param mode linux permissions + * @return true if directory was created + */ + bool mkdir(const std::string& path, const int mode=0733); +}}} diff --git a/interop/util/length_of.h b/interop/util/length_of.h index 0c71fc032..10e091ec7 100644 --- a/interop/util/length_of.h +++ b/interop/util/length_of.h @@ -3,7 +3,6 @@ * This generic function determines the length of an array or vector * * @file - * * @date 10/28/15 * @version 1.0 * @copyright GNU Public License. @@ -58,7 +57,6 @@ namespace illumina { namespace interop { namespace util /** Get the number of elements in a stack array * - * @param val stack array * @return size of array */ template diff --git a/interop/util/lexical_cast.h b/interop/util/lexical_cast.h index 5a1899022..68c22a30f 100644 --- a/interop/util/lexical_cast.h +++ b/interop/util/lexical_cast.h @@ -5,10 +5,9 @@ * the Boost.Lexical_cast. * * @file - * * @date 8/9/15 * @version 1.0 - * @opyright GNU Public License. + * @copyright GNU Public License. */ #pragma once diff --git a/interop/util/linear_hierarchy.h b/interop/util/linear_hierarchy.h index 6eae37e1c..68cd389b7 100644 --- a/interop/util/linear_hierarchy.h +++ b/interop/util/linear_hierarchy.h @@ -10,9 +10,8 @@ #pragma once #include "interop/util/type_traits.h" -namespace illumina { -namespace interop { -namespace hierarchy { +namespace illumina { namespace interop { namespace hierarchy +{ /** Generate a linear hierarchy of classes */ template< class TypeList, @@ -125,6 +124,4 @@ namespace hierarchy { }; -} -} -} +}}} diff --git a/interop/util/math.h b/interop/util/math.h index c1a8b192b..ba4078d3e 100644 --- a/interop/util/math.h +++ b/interop/util/math.h @@ -1,6 +1,6 @@ /** Back port of C++11 math functions * - * @TODO include this everywhere isnan is used + * @todo include this everywhere isnan is used * * @file * @date 4/20/16 @@ -10,9 +10,8 @@ #pragma once -#if defined(HAVE_STD_ISNAN) #include -#else +#if !defined(HAVE_STD_ISNAN) # if defined(HAVE_ISNAN) # include #elif defined(HAVE___ISNAN) diff --git a/interop/util/option_parser.h b/interop/util/option_parser.h index 3c940c642..d4408c2fe 100644 --- a/interop/util/option_parser.h +++ b/interop/util/option_parser.h @@ -16,7 +16,8 @@ #include "interop/util/exception.h" #include "interop/util/unique_ptr.h" -namespace illumina { namespace interop { namespace util{ +namespace illumina { namespace interop { namespace util +{ /** Abstract option */ diff --git a/interop/util/static_assert.h b/interop/util/static_assert.h index 213997a2b..da2d21fef 100644 --- a/interop/util/static_assert.h +++ b/interop/util/static_assert.h @@ -3,7 +3,6 @@ * This is a compile time check for debugging purposes. * * @file - * * @date 9/26/15 * @version 1.0 * @copyright GNU Public License. diff --git a/interop/util/statistics.h b/interop/util/statistics.h index cb9925511..46bd5421a 100644 --- a/interop/util/statistics.h +++ b/interop/util/statistics.h @@ -15,7 +15,6 @@ #include #include #include -#include #include "interop/util/assert.h" #include "interop/util/math.h" @@ -293,7 +292,7 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection + * @param end iterator to end of collection * @param percentile integer in percent * @return iterator to requested percentile */ @@ -316,9 +315,9 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection + * @param end iterator to end of collection * @param percentile integer in percent - * @comp comparator between two types + * @param comp comparator between two types * @return iterator to requested percentile */ template @@ -379,6 +378,7 @@ namespace illumina { namespace interop { namespace util * @param beg iterator to start of sorted array * @param end iterator to end of sorted array * @param percentile target percentile [0-100] + * @param op function that takes one value and returns another value * @return interpolated value of array corresponding to given percentile */ template @@ -407,6 +407,7 @@ namespace illumina { namespace interop { namespace util * * @param beg iterator to start of collection * @param end iterator to end of collection + * @param op function that takes one value and returns another value * @return iterator to last non-NaN element */ template @@ -420,7 +421,7 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection + * @param end iterator to end of collection * @return iterator to requested percentile */ template @@ -434,8 +435,8 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection - * @comp comparator between two types + * @param end iterator to end of collection + * @param comp comparator between two types * @return iterator to requested percentile */ template @@ -448,8 +449,7 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection - * @comp comparator between two types + * @param end iterator to end of collection * @return iterator to requested percentile */ template @@ -463,8 +463,8 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection - * @comp comparator between two types + * @param end iterator to end of collection + * @param comp comparator between two types * @return iterator to requested percentile */ template @@ -478,8 +478,9 @@ namespace illumina { namespace interop { namespace util * @note this will change the underlying array! * * @param beg iterator to start of collection - * @param end iterator to start of collection - * @comp comparator between two types + * @param end iterator to end of collection + * @param comp comparator between two types + * @param op function that takes one value and returns another value * @return iterator to requested percentile */ template @@ -679,6 +680,7 @@ namespace illumina { namespace interop { namespace util * * @param beg iterator to start of collection * @param end iterator to end of collection + * @param mean mean of the data in the collection * @return variance of the input collection */ template diff --git a/interop/util/string.h b/interop/util/string.h index 0faead506..a30f04fe1 100644 --- a/interop/util/string.h +++ b/interop/util/string.h @@ -9,7 +9,8 @@ #pragma once #include -namespace illumina { namespace interop { namespace util { +namespace illumina { namespace interop { namespace util +{ // replace, camel_to_space // diff --git a/interop/util/time.h b/interop/util/time.h index c6c65f4d7..e76c89d6b 100644 --- a/interop/util/time.h +++ b/interop/util/time.h @@ -1,4 +1,3 @@ - /** C# Time Conversion * * @file @@ -8,26 +7,8 @@ */ #pragma once -#include -#include +#include #include "interop/util/cstdint.h" -#include "interop/util/static_assert.h" - -#ifdef HAVE_LONG64 -#define INTEROP_UTIL_TICKS_MASK 0x3fffffffffffffffL -#define INTEROP_UTIL_TICKS_THRESHOLD 0x3fffff36d5964000L -#define INTEROP_UTIL_TICKS_OFFSET 0x4000000000000000L -#define INTEROP_UTIL_TICKS_NEG_OFFSET 0xc92a69c000L -#define INTEROP_UTIL_TICKS_ENCODE 9223372036854775808ul -#define INTEROP_UTIL_TICKS_1970 621355968000000000ul -#else -#define INTEROP_UTIL_TICKS_MASK 0x3fffffffffffffffLL -#define INTEROP_UTIL_TICKS_THRESHOLD 0x3fffff36d5964000LL -#define INTEROP_UTIL_TICKS_OFFSET 0x4000000000000000LL -#define INTEROP_UTIL_TICKS_NEG_OFFSET 0xc92a69c000LL -#define INTEROP_UTIL_TICKS_ENCODE 9223372036854775808ull -#define INTEROP_UTIL_TICKS_1970 621355968000000000ull -#endif namespace illumina { namespace interop { namespace util { @@ -38,23 +19,17 @@ namespace illumina { namespace interop { namespace util struct csharp_date_time { /** Constructor */ - csharp_date_time(uint64_t v = 0) : value(v) { } + csharp_date_time(const ::uint64_t v = 0); /** Convert to time_t unix format * * @return time_t unix format */ - uint64_t to_unix()const - { - return to_unix(value); - } + ::uint64_t to_unix()const; /** Convert to to seconds with fractions * * @return seconds with fractions */ - double to_seconds()const - { - return to_seconds(value); - } + double to_seconds()const; /** Convert to time_t unix format * @@ -63,19 +38,7 @@ namespace illumina { namespace interop { namespace util * @param val C# DateTime.ToBinary format * @return time_t unix format */ - static uint64_t to_unix(uint64_t val) - { - static_assert(sizeof(uint64_t) == 8, "Int64 has the wrong size"); - int64_t ticks = static_cast(val) & INTEROP_UTIL_TICKS_MASK; - if (ticks > INTEROP_UTIL_TICKS_THRESHOLD) - ticks -= INTEROP_UTIL_TICKS_OFFSET; - // TODO: missing conversion to local time (use encoded kind) - if (ticks < 0L) - { - ticks += INTEROP_UTIL_TICKS_NEG_OFFSET; - } - return static_cast( (ticks - ticks_to_1970()) / ticks_per_second() ); - } + static ::uint64_t to_unix(const ::uint64_t val); /** Convert to seconds with fractions * * http://stackoverflow.com/questions/4014652/how-do-datetime-tobinary-and-datetime-tofiletime-differ @@ -83,35 +46,16 @@ namespace illumina { namespace interop { namespace util * @param val C# DateTime.ToBinary format * @return seconds with fractions */ - static double to_seconds(uint64_t val) - { - int64_t ticks = static_cast(val) & INTEROP_UTIL_TICKS_MASK; - if (ticks > INTEROP_UTIL_TICKS_THRESHOLD) - ticks -= INTEROP_UTIL_TICKS_OFFSET; - // TODO: missing conversion to local time (use encoded kind) - if (ticks < 0L) - { - ticks += INTEROP_UTIL_TICKS_NEG_OFFSET; - } - return (ticks - static_cast(ticks_to_1970())) / static_cast(ticks_per_second()); - } + static double to_seconds(const ::uint64_t val); /** Convert to c# format * * @param uval time_t unix format * @return C# DateTime.ToBinary format */ - static csharp_date_time to_csharp(uint64_t uval) - { - int64_t val = static_cast(uval); - val *= ticks_per_second(); - val += ticks_to_1970(); - if(val < 0l) val += INTEROP_UTIL_TICKS_OFFSET; - // TODO: missing conversion to local time (use encoded kind) - return csharp_date_time(static_cast(val | INTEROP_UTIL_TICKS_ENCODE));//-9223372036854775808 - } + static csharp_date_time to_csharp(const ::uint64_t uval); /** Date time in csharp DateTime.ToBinary format */ - uint64_t value; + ::uint64_t value; /** Implicit convertion to an unsigned 64-bit integer * @@ -127,57 +71,31 @@ namespace illumina { namespace interop { namespace util * * @return integer representation */ - uint64_t to_binary()const - { - return value; - } + ::uint64_t to_binary()const; /** Test if the object is equal to another of the same type * * @param other other object being compared * @return true if they are equal */ - bool operator==(const csharp_date_time& other)const - { - const int64_t val = static_cast(value-other.value); - return ((val > 0) ? val : -val) < 5e14; - } + bool operator==(const csharp_date_time& other)const; /** Write date type as integer to an output stream * * @param out output stream * @param date_time data time object * @return output stream */ - friend std::ostream& operator<<(std::ostream& out, const csharp_date_time& date_time ) - { - // TODO: write out nice string representation - //2015-02-25 20:04:00 - out << date_time.value; - return out; - } + friend std::ostream& operator<<(std::ostream& out, const csharp_date_time& date_time ); /** Read a date type from the input stream * * @param in input stream * @param date_time data time object * @return input stream */ - friend std::istream& operator>>(std::istream& in, csharp_date_time& date_time) - { - // TODO: read in nice string representation - uint64_t val; - in >> val; - date_time = csharp_date_time(val); - return in; - } + friend std::istream& operator>>(std::istream& in, csharp_date_time& date_time); private: - inline static ::uint64_t ticks_per_second() - { - return 10000000; - } - inline static ::uint64_t ticks_to_1970() - { - return INTEROP_UTIL_TICKS_1970; - } + static ::uint64_t ticks_per_second(); + static ::uint64_t ticks_to_1970(); }; #pragma pack() }}} diff --git a/interop/util/type_traits.h b/interop/util/type_traits.h index ad00e8efe..fc2d3efac 100644 --- a/interop/util/type_traits.h +++ b/interop/util/type_traits.h @@ -9,8 +9,8 @@ */ #pragma once -namespace illumina { -namespace interop { +namespace illumina { namespace interop +{ /** Place holder for */ struct null_type{}; @@ -29,12 +29,14 @@ namespace interop { typename T1=null_type, typename T2=null_type, typename T3=null_type, typename T4=null_type, typename T5=null_type, typename T6=null_type, typename T7=null_type, typename T8=null_type, typename T9=null_type, typename T10=null_type, typename T11=null_type, typename T12=null_type, - typename T13=null_type, typename T14=null_type, typename T15=null_type, typename T16=null_type> + typename T13=null_type, typename T14=null_type, typename T15=null_type, typename T16=null_type, + typename T17=null_type, typename T18=null_type, typename T19=null_type, typename T20=null_type, + typename T21=null_type, typename T22=null_type, typename T23=null_type, typename T24=null_type> struct make_type_list { private: typedef typename make_type_list< - T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16 + T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24 >::result_t tail_result_t; public: /** List of types */ @@ -120,7 +122,7 @@ namespace interop { /** Define the value type */ typedef T value_t; /** Define the pointer type */ - typedef T* pointer_t; + typedef constant_type* pointer_t; /** Get the value of the constant type * * @return value @@ -146,5 +148,4 @@ namespace interop { */ template struct int_constant_type : public constant_type{}; -} -} +}} diff --git a/interop/util/unique_ptr.h b/interop/util/unique_ptr.h index f60092d2d..70756b5c3 100644 --- a/interop/util/unique_ptr.h +++ b/interop/util/unique_ptr.h @@ -2,7 +2,6 @@ * * * @file - * * @date 8/6/2015 * @version 1.0 * @copyright GNU Public License. diff --git a/interop/util/xml_exceptions.h b/interop/util/xml_exceptions.h index 964d20a15..4dfa5d642 100644 --- a/interop/util/xml_exceptions.h +++ b/interop/util/xml_exceptions.h @@ -7,84 +7,94 @@ */ #pragma once + #include -#include "interop/util/exception_specification.h" -namespace illumina -{ -namespace interop -{ -namespace xml +namespace illumina { namespace interop { namespace xml { -/** Base exception for all format errors - */ -struct xml_format_exception : public std::runtime_error { - /** Constructor - * - * @param mesg error message + /** Base exception for all format errors */ - xml_format_exception(const std::string &mesg) : std::runtime_error(mesg) { } -}; -/** @defgroup xml_exceptions XML Parsing Exceptions - * - * Exceptions that will be thrown if a problem occurs while reading an XML file. - * - * @ingroup interop_exceptions - * @{ - */ -/** Exception raised if the XML file is not found in the file system - */ -struct xml_file_not_found_exception : public std::runtime_error { - /** Constructor + struct xml_format_exception : public std::runtime_error + { + /** Constructor + * + * @param mesg error message + */ + xml_format_exception(const std::string &mesg) : std::runtime_error(mesg) + { } + }; + /** @defgroup xml_exceptions XML Parsing Exceptions * - * @param mesg error message - */ - xml_file_not_found_exception(const std::string &mesg) : std::runtime_error(mesg) {} -}; -/** Exception raised if the XML does not have the correct format - * - * This is only raised if the format is bad. - */ -struct xml_parse_exception : public xml_format_exception { - /** Constructor + * Exceptions that will be thrown if a problem occurs while reading an XML file. * - * @param mesg error message + * @ingroup interop_exceptions + * @{ */ - xml_parse_exception(const std::string &mesg) : xml_format_exception(mesg) { } -}; -/** Exception raised if the XML does not have the correct format - * - * This is only raised if the format is bad. - */ -struct bad_xml_format_exception : public xml_format_exception { - /** Constructor - * - * @param mesg error message + /** Exception raised if the XML file is not found in the file system */ - bad_xml_format_exception(const std::string &mesg) : xml_format_exception(mesg) { } -}; -/** Exception raised if the XML file record is empty - */ -struct empty_xml_format_exception : public xml_format_exception { - /** Constructor + struct xml_file_not_found_exception : public std::runtime_error + { + /** Constructor + * + * @param mesg error message + */ + xml_file_not_found_exception(const std::string &mesg) : std::runtime_error(mesg) + { } + }; + + /** Exception raised if the XML does not have the correct format * - * @param mesg error message + * This is only raised if the format is bad. */ - empty_xml_format_exception(const std::string &mesg) : xml_format_exception(mesg) { } -}; -/** Exception raised if the XML file is missing a required element - */ -struct missing_xml_element_exception : public xml_format_exception { - /** Constructor + struct xml_parse_exception : public xml_format_exception + { + /** Constructor + * + * @param mesg error message + */ + xml_parse_exception(const std::string &mesg) : xml_format_exception(mesg) + { } + }; + + /** Exception raised if the XML does not have the correct format * - * @param mesg error message + * This is only raised if the format is bad. + */ + struct bad_xml_format_exception : public xml_format_exception + { + /** Constructor + * + * @param mesg error message + */ + bad_xml_format_exception(const std::string &mesg) : xml_format_exception(mesg) + { } + }; + + /** Exception raised if the XML file record is empty + */ + struct empty_xml_format_exception : public xml_format_exception + { + /** Constructor + * + * @param mesg error message + */ + empty_xml_format_exception(const std::string &mesg) : xml_format_exception(mesg) + { } + }; + + /** Exception raised if the XML file is missing a required element */ - missing_xml_element_exception(const std::string &mesg) : xml_format_exception(mesg) { } -}; + struct missing_xml_element_exception : public xml_format_exception + { + /** Constructor + * + * @param mesg error message + */ + missing_xml_element_exception(const std::string &mesg) : xml_format_exception(mesg) + { } + }; -/** @} */ + /** @} */ -} -} -} +}}} diff --git a/interop/util/xml_parser.h b/interop/util/xml_parser.h index da2186430..4dd82fc2d 100644 --- a/interop/util/xml_parser.h +++ b/interop/util/xml_parser.h @@ -8,165 +8,167 @@ */ #pragma once + #include #include #include "interop/util/xml_exceptions.h" #include "interop/util/exception.h" -namespace illumina -{ -namespace interop -{ -namespace xml +namespace illumina { namespace interop { namespace xml { -/** XML node pointer type */ -typedef rapidxml::xml_node<>* xml_node_ptr; -/** XML attribute pointer type */ -typedef rapidxml::xml_attribute<> *xml_attr_ptr; -/** Check if the xml node matches the target, and, if so, set the value - * - * @param p_node current node - * @param target target string - * @param val destination value - * @param default_val default value if the tag is not found - * @return true if the target was found - */ -template -bool set_data_with_default(xml_node_ptr p_node, const std::string &target, T &val, const T default_val) -{ - if(p_node == 0) + /** XML node pointer type */ + typedef rapidxml::xml_node<> *xml_node_ptr; + /** XML attribute pointer type */ + typedef rapidxml::xml_attribute<> *xml_attr_ptr; + + /** Check if the xml node matches the target, and, if so, set the value + * + * @param p_node current node + * @param target target string + * @param val destination value + * @param default_val default value if the tag is not found + * @return true if the target was found + */ + template + bool set_data_with_default(xml_node_ptr p_node, const std::string &target, T &val, const T default_val) { - val = default_val; - return false; + if (p_node == 0) + { + val = default_val; + return false; + } + if (p_node->name() == target) + { + val = util::lexical_cast(p_node->value()); + return true; + } + INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); } - if (p_node->name() == target) + + /** Find the target node and set the value + * + * @param p_node current node + * @param target target string + * @param val destination value + * @return true if the target was found + */ + template + void set_data_for_target(xml_node_ptr p_node, const std::string &target, T &val) { + if (p_node == 0) + INTEROP_THROW(missing_xml_element_exception, "Cannot find parent"); + p_node = p_node->first_node(target.c_str()); + if (p_node == 0) + INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); val = util::lexical_cast(p_node->value()); - return true; } - INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); -} -/** Find the target node and set the value - * - * @param p_node current node - * @param target target string - * @param val destination value - * @return true if the target was found - */ -template -void set_data_for_target(xml_node_ptr p_node, const std::string &target, T &val) -{ - if (p_node == 0) - INTEROP_THROW(missing_xml_element_exception, "Cannot find parent"); - p_node = p_node->first_node(target.c_str()); - if (p_node == 0) - INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); - val = util::lexical_cast(p_node->value()); -} -/** Find the target node and set the value - * - * @param p_node current node - * @param target target string - * @param val destination value - * @param default_val default value if the tag is not found - * @return true if the target was found - */ -template -void set_data_for_target(xml_node_ptr p_node, const std::string &target, T &val, const T default_val) -{ - if (p_node == 0) INTEROP_THROW( missing_xml_element_exception, "Cannot find parent"); - p_node = p_node->first_node(target.c_str()); - if (p_node == 0) val = default_val; - else val = util::lexical_cast(p_node->value()); -} + /** Find the target node and set the value + * + * @param p_node current node + * @param target target string + * @param val destination value + * @param default_val default value if the tag is not found + * @return true if the target was found + */ + template + void set_data_for_target(xml_node_ptr p_node, const std::string &target, T &val, const T default_val) + { + if (p_node == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find parent"); + p_node = p_node->first_node(target.c_str()); + if (p_node == 0) val = default_val; + else val = util::lexical_cast(p_node->value()); + } -/** Check if the xml node matches the target, and, if so, set the value - * - * @param p_node current node - * @param target target string - * @param val destination value - * @return true if the target was found - */ -template -bool set_data(xml_node_ptr p_node, const std::string &target, T &val) -{ - if (p_node == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); - if (p_node->name() == target) + /** Check if the xml node matches the target, and, if so, set the value + * + * @param p_node current node + * @param target target string + * @param val destination value + * @return true if the target was found + */ + template + bool set_data(xml_node_ptr p_node, const std::string &target, T &val) { - val = util::lexical_cast(p_node->value()); - return true; + if (p_node == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); + if (p_node->name() == target) + { + val = util::lexical_cast(p_node->value()); + return true; + } + return false; } - return false; -} -/** Convert the value of the node to the destination type - * - * @param p_attr current node - * @param val destination value - */ -template -void set_data(xml_node_ptr p_attr, T &val) -{ - if (p_attr == 0) INTEROP_THROW( missing_xml_element_exception, "Cannot find node"); - val = util::lexical_cast(p_attr->value()); -} -/** Check if the xml attributes matches the target, and, if so, set the value - * - * @param p_attr current attribute - * @param target target string - * @param val destination value - * @return true if the target was found - */ -template -bool set_data(xml_attr_ptr p_attr, const std::string &target, T &val) -{ - if (p_attr == 0) INTEROP_THROW( missing_xml_element_exception, "Cannot find attribute: " << target); - if (p_attr->name() == target) + + /** Convert the value of the node to the destination type + * + * @param p_attr current node + * @param val destination value + */ + template + void set_data(xml_node_ptr p_attr, T &val) { + if (p_attr == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find node"); val = util::lexical_cast(p_attr->value()); - return true; } - return false; -} -/** Convert the value of the attribute to the destination type - * - * @param p_attr current attribute - * @param val destination value - */ -template -void set_data(xml_attr_ptr p_attr, T &val) -{ - if (p_attr == 0) - INTEROP_THROW(missing_xml_element_exception, "Cannot find attribute"); - val = util::lexical_cast(p_attr->value()); -} -/** Check if the xml node matches the target, and, if so, save the children to the vector - * - * @param p_node current node - * @param target target string - * @param val destination vector - * @return true if the target was found - */ -template -bool set_data(xml_node_ptr p_node, const std::string &target, const std::string& child, std::vector &val) -{ - if (p_node == 0) - INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); - if (p_node->name() == target) + + /** Check if the xml attributes matches the target, and, if so, set the value + * + * @param p_attr current attribute + * @param target target string + * @param val destination value + * @return true if the target was found + */ + template + bool set_data(xml_attr_ptr p_attr, const std::string &target, T &val) + { + if (p_attr == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find attribute: " << target); + if (p_attr->name() == target) + { + val = util::lexical_cast(p_attr->value()); + return true; + } + return false; + } + + /** Convert the value of the attribute to the destination type + * + * @param p_attr current attribute + * @param val destination value + */ + template + void set_data(xml_attr_ptr p_attr, T &val) { - val.clear(); - for (rapidxml::xml_node<> *p_name = p_node->first_node(); p_name; p_name = p_name->next_sibling()) + if (p_attr == 0) + INTEROP_THROW(missing_xml_element_exception, "Cannot find attribute"); + val = util::lexical_cast(p_attr->value()); + } + + /** Check if the xml node matches the target, and, if so, save the children to the vector + * + * @param p_node current node + * @param target target string + * @param child child node name + * @param val destination vector + * @return true if the target was found + */ + template + bool set_data(xml_node_ptr p_node, const std::string &target, const std::string &child, std::vector &val) + { + if (p_node == 0) + INTEROP_THROW(missing_xml_element_exception, "Cannot find node: " << target); + if (p_node->name() == target) { - if(p_name->name() != child) continue; - val.push_back(util::lexical_cast(p_name->value())); + val.clear(); + for (rapidxml::xml_node<> *p_name = p_node->first_node(); p_name; p_name = p_name->next_sibling()) + { + if (p_name->name() != child) continue; + val.push_back(util::lexical_cast(p_name->value())); + } + return true; } - return true; + return false; } - return false; -} -} -} -} +}}} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8ec3133b3..e879f8a28 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,4 +8,7 @@ endif() add_subdirectory("apps") add_subdirectory("examples") -add_subdirectory("tests") + +if(ENABLE_TEST) + add_subdirectory("tests") +endif() diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt index 9b68c27df..c27f5cca7 100644 --- a/src/apps/CMakeLists.txt +++ b/src/apps/CMakeLists.txt @@ -21,6 +21,10 @@ function(add_application _target _source_files) endif() endfunction() +if(MSVC) + add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS) +endif() + add_application(interop2csv interop2csv.cpp) add_application(cyclesim cyclesim.cpp) add_application(summary summary.cpp) diff --git a/src/apps/cyclesim.cpp b/src/apps/cyclesim.cpp index 61a7b029d..25948ef5b 100644 --- a/src/apps/cyclesim.cpp +++ b/src/apps/cyclesim.cpp @@ -1,5 +1,4 @@ /** @page cyclesim Cycle Simulator - * * * This application writes out a new set of binary InterOp files for all records up to a specific cycle. * @@ -68,34 +67,34 @@ void print_help(std::ostream& out); * 5. Image * 6. Q * 7. Index - * - * @param input path to run folder + + * @param filename path to run folder * @param output path to output run folder - * @param max_cycle number of cycles to copy - * @param max_read number of reads to copy - * @param cycle_to_align cycle alignment occurs - * @return error code or 0 + * @param max_cycle maximum number of cycles + * @param max_read maximum number of reads + * @param cycle_to_align cycle to align + * @return 0 if success, or an error code */ -int write_interops(const std::string& input, const std::string& output, unsigned int max_cycle, unsigned int max_read, unsigned int cycle_to_align); +int write_interops(const std::string& filename, + const std::string& output, + const unsigned int max_cycle, + const unsigned int max_read, + const unsigned int cycle_to_align); -/** Set false if you want to disable error messages printing to the error stream */ -bool kPrintError=true; -/** Set greater than zero if you want to view less recoreds */ -int kMaxRecordCount=0; int main(int argc, char** argv) { if(argc <= 1) { - if(kPrintError) std::cerr << "No arguments specified!" << std::endl; - if(kPrintError) print_help(std::cout); + std::cerr << "No arguments specified!" << std::endl; + print_help(std::cout); return 1; } if(argc < 5) { - if(kPrintError) std::cerr << "Too few arguments specified!" << std::endl; - if(kPrintError) print_help(std::cout); + std::cerr << "Too few arguments specified!" << std::endl; + print_help(std::cout); return 1; } @@ -154,19 +153,19 @@ int read_metrics_from_file(const std::string& filename, MetricSet& metrics) catch(const io::file_not_found_exception&){return 1;} catch(const io::bad_format_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return BAD_FORMAT; } catch(const io::incomplete_file_exception&){ } catch(const std::exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return UNEXPECTED_EXCEPTION; } if(metrics.size()==0) { - if(kPrintError) std::cerr << "Empty metric file: " << io::interop_basename() << std::endl; + std::cerr << "Empty metric file: " << io::interop_basename() << std::endl; return EMPTY_INTEROP; } return 0; @@ -195,12 +194,12 @@ int copy_tile_metrics(const std::string& input, const std::string& output, const } catch(const io::file_not_found_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return 1; } catch(const io::bad_format_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return BAD_FORMAT; } @@ -218,18 +217,18 @@ int copy_tile_metrics(const std::string& input, const std::string& output, const subset.push_back(tile_metric(*beg, reads)); } - tile_metric_set_t metricsOut(subset, metrics.version(), metrics); + tile_metric_set_t metrics_out(subset, metrics.version(), metrics); try { - io::write_interop(output, metricsOut); + io::write_interop(output, metrics_out); } catch(const io::file_not_found_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return 1; } catch(const io::bad_format_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return BAD_FORMAT; } @@ -269,18 +268,18 @@ int copy_cycle_metrics(const std::string& input, const std::string& output, cons subset.push_back(*beg); } - MetricSet metricsOut(subset, metrics.version(), metrics); + MetricSet metrics_out(subset, metrics.version(), metrics); try { - io::write_interop(output, metricsOut); + io::write_interop(output, metrics_out); } catch(const io::file_not_found_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return 1; } catch(const io::bad_format_exception& ex) { - if(kPrintError) std::cerr << ex.what() << std::endl; + std::cerr << ex.what() << std::endl; return BAD_FORMAT; } @@ -299,10 +298,11 @@ int encode_error(const int res, const int type) } /** Encode error type and metric type into a single code * - * @param input path to run folder + * @param filename path to run folder * @param output path to output run folder * @param max_cycle maximum number of cycles * @param max_read maximum number of reads + * @param cycle_to_align cycle to align * @return 0 if success, or an error code */ int write_interops(const std::string& filename, @@ -330,7 +330,7 @@ int write_interops(const std::string& filename, if(res == 0) valid_count++; if(valid_count == 0) { - if(kPrintError) std::cerr << "No files found" << std::endl; + std::cerr << "No files found" << std::endl; return EMPTY_INTEROP; } return 0; diff --git a/src/apps/imaging_table.cpp b/src/apps/imaging_table.cpp index 47d1c42cf..a412cfa87 100644 --- a/src/apps/imaging_table.cpp +++ b/src/apps/imaging_table.cpp @@ -23,7 +23,8 @@ #include #include "interop/io/metric_file_stream.h" #include "interop/model/run_metrics.h" -#include "interop/logic/table/populate_imaging_table.h" +#include "interop/logic/table/create_imaging_table.h" +#include "interop/io/table/imaging_table_csv.h" #include "interop/version.h" #include "inc/application.h" @@ -49,15 +50,13 @@ int main(int argc, char** argv) model::table::imaging_table table; try { - logic::table::populate_imaging_table(run, table); - logic::table::populate_column_headers(run.run_info().channels(), table); + logic::table::create_imaging_table(run, table); } catch(const std::exception& ex) { std::cerr << ex.what() << std::endl; return UNEXPECTED_EXCEPTION; } - //table.resize(3); std::cout << table << std::endl; } return SUCCESS; diff --git a/src/apps/interop2csv.cpp b/src/apps/interop2csv.cpp index 5343f36c7..02a8e7e05 100644 --- a/src/apps/interop2csv.cpp +++ b/src/apps/interop2csv.cpp @@ -207,13 +207,13 @@ int read_metrics_from_file(const std::string& filename, MetricSet& metrics) try { io::read_interop(filename, metrics); } - catch(const io::file_not_found_exception&){return 1;} + catch(const io::file_not_found_exception& ex){if(kPrintError) std::cerr << ex.what() << std::endl;return 1;} catch(const io::bad_format_exception& ex) { if(kPrintError) std::cerr << io::interop_basename() << " - " << ex.what() << std::endl; return BAD_FORMAT; } - catch(const io::incomplete_file_exception&){ } + catch(const io::incomplete_file_exception& ex){ if(kPrintError) std::cerr << ex.what() << std::endl; } catch(const std::exception& ex) { if(kPrintError) std::cerr << io::interop_basename() << " - " << ex.what() << std::endl; @@ -313,7 +313,7 @@ int write_tile_metrics(std::ostream& out, const std::string& filename) { const tile_metric& metric = *beg; write_id(out, metric); - out << metric.clusterCount() << "," << metric.clusterCountPf() << "," << metric.clusterDensity() << "," << metric.clusterDensityPf() << "\n"; + out << metric.cluster_count() << "," << metric.cluster_count_pf() << "," << metric.cluster_density() << "," << metric.cluster_density_pf() << "\n"; } write_header(out, metrics); @@ -360,7 +360,7 @@ int write_error_metrics(std::ostream& out, const std::string& filename) { const error_metric& metric = *beg; write_id(out, metric); - out << metric.errorRate() << "\n"; + out << metric.error_rate() << "\n"; } return res; @@ -399,13 +399,13 @@ int write_corrected_intensity_metrics(std::ostream& out, const std::string& file { const corrected_intensity_metric& metric = *beg; write_id(out, metric); - out << metric.averageCycleIntensity() << "," << metric.signalToNoise(); + out << metric.average_cycle_intensity() << "," << metric.signal_to_noise(); for(int i=-1;i(i)); for(size_t i=0;i(i)); for(size_t i=0;i(i)); out << "\n"; } @@ -443,11 +443,11 @@ int write_extraction_metrics(std::ostream& out, const std::string& filename) thr { const extraction_metric& metric = *beg; write_id(out, metric); - out << metric.dateTime(); + out << metric.date_time(); for(size_t i=0;i0) + if(metrics.bin_count()>0) { write_header(out, metrics); out << "Type"; - for (size_t i = 0; i < metrics.binCount(); i++) + for (size_t i = 0; i < metrics.bin_count(); i++) out << ",Bin" << i + 1; out << "\n"; out << "Lower"; - for (size_t i = 0; i < metrics.binCount(); i++) - out << "," << metrics.binAt(i).lower(); + for (size_t i = 0; i < metrics.bin_count(); i++) + out << "," << metrics.bin_at(i).lower(); out << "\n"; out << "Upper"; - for (size_t i = 0; i < metrics.binCount(); i++) - out << "," << metrics.binAt(i).upper(); + for (size_t i = 0; i < metrics.bin_count(); i++) + out << "," << metrics.bin_at(i).upper(); out << "\n"; out << "Value"; - for (size_t i = 0; i < metrics.binCount(); i++) - out << "," << metrics.binAt(i).value(); + for (size_t i = 0; i < metrics.bin_count(); i++) + out << "," << metrics.bin_at(i).value(); out << "\n"; } @@ -595,7 +595,7 @@ int write_q_metrics(std::ostream& out, const std::string& filename) write_id(out, metric); out << metric.size(); for(size_t i=0;i0) + if(metrics.bin_count()>0) { write_header(out, metrics); out << "Type"; - for (size_t i = 0; i < metrics.binCount(); i++) + for (size_t i = 0; i < metrics.bin_count(); i++) out << ",Bin" << i + 1; out << "\n"; out << "Lower"; - for (size_t i = 0; i < metrics.binCount(); i++) - out << "," << metrics.binAt(i).lower(); + for (size_t i = 0; i < metrics.bin_count(); i++) + out << "," << metrics.bin_at(i).lower(); out << "\n"; out << "Upper"; - for (size_t i = 0; i < metrics.binCount(); i++) - out << "," << metrics.binAt(i).upper(); + for (size_t i = 0; i < metrics.bin_count(); i++) + out << "," << metrics.bin_at(i).upper(); out << "\n"; out << "Value"; - for (size_t i = 0; i < metrics.binCount(); i++) - out << "," << metrics.binAt(i).value(); + for (size_t i = 0; i < metrics.bin_count(); i++) + out << "," << metrics.bin_at(i).value(); out << "\n"; } @@ -665,7 +665,7 @@ int write_q_by_lane_metrics(std::ostream& out, const std::string& filename) write_id(out, metric); out << metric.size(); for(size_t i=0;i) -set(EXAMPLE_SRCS - Example2.cs - ) +function(add_csharp_example _target _source_files) + set(_abs_source_files) + foreach(SRC ${_source_files}) + list(APPEND _abs_source_files "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}") + endforeach() + csharp_add_executable(${_target} ${_abs_source_files} ${SWIG_CSHARP_LIBRARY}) + add_dependencies(${_target} csharp_interop) + add_custom_command(TARGET ${_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${CSHARP_${_target}_BINARY} $) + if(NOT ENABLE_EXAMPLES) + set_target_properties(${_target} PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) + else() + install(FILES ${CSHARP_${_target}_BINARY} ${CSHARP_INTEROP_LIBRARY} + DESTINATION share/illumina/interop/examples + ) + endif() + if(NOT ENABLE_STATIC) + add_custom_command(TARGET ${_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${CMAKE_CURRENT_BINARY_DIR}) + endif() +endfunction() + +add_csharp_example(csharp_example1 Example1.cs) +add_csharp_example(csharp_example2 Example2.cs) +add_csharp_example(csharp_example3 Example3.cs) -set(csharp_files ${SWIG_GEN_CSHARP_SOURCE_FILES}) -foreach(SRC ${EXAMPLE_SRCS}) - get_filename_component(CSHARP_TEST_SRCS ${SRC} ABSOLUTE) - list(APPEND csharp_files ${CSHARP_TEST_SRCS}) -endforeach() - -csharp_add_executable(csharp_example2 ${csharp_files}) -add_dependencies(csharp_example2 csharp_interop) -add_custom_command(TARGET csharp_example2 POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CSHARP_csharp_example2_BINARY} $) - - -set(EXAMPLE_SRCS - Example3.cs - ) - -set(csharp_files ${SWIG_GEN_CSHARP_SOURCE_FILES}) -foreach(SRC ${EXAMPLE_SRCS}) - get_filename_component(CSHARP_TEST_SRCS ${SRC} ABSOLUTE) - list(APPEND csharp_files ${CSHARP_TEST_SRCS}) -endforeach() - -csharp_add_executable(csharp_example3 ${csharp_files}) -add_dependencies(csharp_example3 csharp_interop) -add_custom_command(TARGET csharp_example3 POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CSHARP_csharp_example3_BINARY} $) - -if(NOT ENABLE_EXAMPLES) - set_target_properties(csharp_example1 PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) - set_target_properties(csharp_example2 PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) - set_target_properties(csharp_example3 PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) -else() - install(FILES ${CSHARP_csharp_example1_BINARY} ${CSHARP_csharp_example2_BINARY} ${CSHARP_csharp_example3_BINARY} ${CSHARP_INTEROP_LIBRARY} - DESTINATION share/illumina/interop/examples - ) -endif() - -if(NOT ENABLE_STATIC) - add_custom_command(TARGET csharp_example1 POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${CMAKE_CURRENT_BINARY_DIR}) - add_custom_command(TARGET csharp_example2 POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${CMAKE_CURRENT_BINARY_DIR}) - add_custom_command(TARGET csharp_example3 POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${CMAKE_CURRENT_BINARY_DIR}) -endif() - - -if(NOT ENABLE_EXAMPLES) - set_target_properties(csharp_example1 PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) - set_target_properties(csharp_example2 PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) - set_target_properties(csharp_example3 PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) -endif() diff --git a/src/examples/csharp/Example1.cs b/src/examples/csharp/Example1.cs index c569e0cb9..0fc4874f8 100644 --- a/src/examples/csharp/Example1.cs +++ b/src/examples/csharp/Example1.cs @@ -3,6 +3,7 @@ using System; using Illumina.InterOp.Run; using Illumina.InterOp.Metrics; +using Illumina.InterOp.Comm; class Example1 { @@ -21,7 +22,7 @@ static int Main(string[] args) base_tile_metrics tile_metric_set = new base_tile_metrics (); try { - c_csharp_metrics.read_interop (args [0], tile_metric_set); + c_csharp_comm.read_interop (args [0], tile_metric_set); } catch(incomplete_file_exception){} catch(file_not_found_exception ex) diff --git a/src/examples/csharp/Example2.cs b/src/examples/csharp/Example2.cs index de48fcac9..ab9869c83 100644 --- a/src/examples/csharp/Example2.cs +++ b/src/examples/csharp/Example2.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using Illumina.InterOp.Run; using Illumina.InterOp.Metrics; +using Illumina.InterOp.Comm; class TileSummary @@ -62,7 +63,7 @@ public static int ReadInterop(string filename, base_tile_metrics tile_metric_set { try { - c_csharp_metrics.read_interop (filename, tile_metric_set); + c_csharp_comm.read_interop (filename, tile_metric_set); } catch(incomplete_file_exception){} catch(file_not_found_exception ex) diff --git a/src/examples/csharp/Example3.cs b/src/examples/csharp/Example3.cs index 8326357f1..cc30d1574 100644 --- a/src/examples/csharp/Example3.cs +++ b/src/examples/csharp/Example3.cs @@ -4,6 +4,7 @@ using System.Linq; using Illumina.InterOp.Run; using Illumina.InterOp.Metrics; +using Illumina.InterOp.Comm; class Example3 @@ -41,7 +42,7 @@ public static int ReadInterop(string filename, base_extraction_metrics extractio { try { - c_csharp_metrics.read_interop (filename, extraction_metric_set); + c_csharp_comm.read_interop (filename, extraction_metric_set); } catch(incomplete_file_exception){} catch(file_not_found_exception ex) diff --git a/src/examples/example2.cpp b/src/examples/example2.cpp index 784deb8db..d4a348067 100644 --- a/src/examples/example2.cpp +++ b/src/examples/example2.cpp @@ -44,8 +44,8 @@ int main(int argc, char** argv) lane_summary_map_t lane_summary_map; for(metric_set::metric_array_t::const_iterator beg = tile_metric_set.metrics().begin(), end = tile_metric_set.metrics().end();beg != end;++beg) { - lane_summary_map[beg->lane()].cluster_count += beg->clusterCount(); - lane_summary_map[beg->lane()].cluster_count_pf += beg->clusterCountPf(); + lane_summary_map[beg->lane()].cluster_count += beg->cluster_count(); + lane_summary_map[beg->lane()].cluster_count_pf += beg->cluster_count_pf(); lane_summary_map[beg->lane()].tile_count += 1; } diff --git a/src/examples/example3.cpp b/src/examples/example3.cpp index ffb0c3acd..22a9480b4 100644 --- a/src/examples/example3.cpp +++ b/src/examples/example3.cpp @@ -32,7 +32,7 @@ int main(int argc, char** argv) if((ret = read_interop_file(argv[1], extraction_metric_set)) != 0) return ret; - std::time_t t = static_cast(extraction_metric_set.metrics()[0].dateTime()); + std::time_t t = static_cast(extraction_metric_set.metrics()[0].date_time()); std::tm* tm = std::gmtime(&t); if(tm != 0) { diff --git a/src/examples/example4.cpp b/src/examples/example4.cpp index a8fefd231..78cb84449 100644 --- a/src/examples/example4.cpp +++ b/src/examples/example4.cpp @@ -55,7 +55,7 @@ int main(int argc, char** argv) // Member function that takes no arguments float avg_no_call = mean(corrected_intensity_metric_set.metrics().begin(), corrected_intensity_metric_set.metrics().end(), - op::const_member_function(&corrected_intensity_metric::noCalls)); + op::const_member_function(&corrected_intensity_metric::no_calls)); std::cout << "Mean of percent base for A " << avg << std::endl; std::cout << "Standard Deviation of percent base for A " << standard_dev << std::endl; std::cout << "Mean of no call" << avg_no_call << std::endl; diff --git a/src/ext/CMakeLists.txt b/src/ext/CMakeLists.txt index ceaadf617..30e847add 100644 --- a/src/ext/CMakeLists.txt +++ b/src/ext/CMakeLists.txt @@ -21,11 +21,15 @@ if(IS_INT64_LONG) else() set(SWIG_WORDSIZE_FLAG "") endif() +if("${SIZE_T}" EQUAL 8) + set(SWIG_WORDSIZE_FLAG "${SWIG_WORDSIZE_FLAG};-DHAVE_UINT64_SIZE_T") +endif() include(${SWIG_USE_FILE}) set(CMAKE_SWIG_FLAGS "") -set(SWIG_SRCS_TEMP swig/run.i swig/metrics.i swig/imaging.i swig/plot.i swig/summary.i) +set(SWIG_SRCS_TEMP swig/run.i swig/metrics.i swig/comm.i swig/imaging.i swig/plot.i swig/summary.i) set(SWIG_DEPS_TEMP + swig/util/operator_overload.i swig/exceptions/exceptions_impl.i swig/exceptions/exceptions_csharp.i swig/arrays/arrays_csharp_impl.i @@ -54,6 +58,9 @@ endforeach() # Language options # https://github.com/SimpleITK/SimpleITK/blob/master/CMake/sitkLanguageOptions.cmake -add_subdirectory("csharp") + +if(ENABLE_CSHARP) + add_subdirectory("csharp") +endif() diff --git a/src/ext/csharp/CMakeLists.txt b/src/ext/csharp/CMakeLists.txt index 826166964..304bdc848 100644 --- a/src/ext/csharp/CMakeLists.txt +++ b/src/ext/csharp/CMakeLists.txt @@ -1,8 +1,11 @@ # Build a C# Shared library using SWIG -set(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_SOURCE_DIR}/src) -set(SWIG_MODULE_c_csharp_interop_EXTRA_DEPS ${SWIG_DEPS} ) +set(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_BINARY_DIR}) + + +add_custom_target(csharp_lib) +set_target_properties(csharp_lib PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) foreach(SRC ${SWIG_SRCS}) get_filename_component(MODULE ${SRC} NAME_WE) @@ -11,6 +14,7 @@ foreach(SRC ${SWIG_SRCS}) string(REGEX REPLACE "^.(.*)" "${FIRST_LETTER}\\1" NAMESPACE "${MODULE}") set_source_files_properties(${SRC} PROPERTIES SWIG_FLAGS "-namespace;Illumina.InterOp.${NAMESPACE};-module;c_csharp_${MODULE}${SWIG_WORDSIZE_FLAG}") set_source_files_properties(${SRC} PROPERTIES CPLUSPLUS ON) + set(SWIG_MODULE_c_csharp_${MODULE}_EXTRA_DEPS ${SWIG_DEPS} ) swig_add_module(c_csharp_${MODULE} csharp ${SRC}) set(SWIG_MODULE ${SWIG_MODULE_c_csharp_${MODULE}_REAL_NAME}) set(EXTRA_LINKER_FLAGS) @@ -23,11 +27,11 @@ foreach(SRC ${SWIG_SRCS}) set_target_properties(${SWIG_MODULE} PROPERTIES LINK_FLAGS "${EXTRA_LINKER_FLAGS}") endif() if(MSVC) - set_target_properties(${SWIG_MODULE} PROPERTIES COMPILE_FLAGS "/bigobj") + set_target_properties(${SWIG_MODULE} PROPERTIES COMPILE_FLAGS "/bigobj /wd4100 -D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS") elseif(MINGW) - set_target_properties(${SWIG_MODULE} PROPERTIES COMPILE_FLAGS "-Wa,-mbig-obj -Wno-unused-function") + set_target_properties(${SWIG_MODULE} PROPERTIES COMPILE_FLAGS "-Wa,-mbig-obj -Wno-unused-function -Wno-unused-parameter") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR CMAKE_COMPILER_IS_GNUCC) - set_target_properties(${SWIG_MODULE} PROPERTIES COMPILE_FLAGS "-Wno-unused-function") + set_target_properties(${SWIG_MODULE} PROPERTIES COMPILE_FLAGS "-Wno-unused-function -Wno-unused-parameter") endif() swig_link_libraries(${SWIG_MODULE} ${INTEROP_DL_LIB}) @@ -41,9 +45,11 @@ foreach(SRC ${SWIG_SRCS}) COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${CMAKE_CURRENT_BINARY_DIR}) endif() list(APPEND SWIG_MODULE_LIST ${SWIG_MODULE}) + add_dependencies(csharp_lib ${SWIG_MODULE}) endforeach() + find_package(CSharp) if(NOT CSHARP_FOUND) @@ -67,6 +73,7 @@ foreach(SRC ${SWIG_SRCS}) COMMENT "copy ${CSHARP_csharp_interop_BINARY} $" ) endforeach() +set(SWIG_CSHARP_LIBRARY ${CSHARP_csharp_interop_BINARY} CACHE INTERNAL "C# library generated by SWIG" FORCE) if(ENABLE_EXAMPLES) foreach(SRC ${SWIG_SRCS}) @@ -83,3 +90,9 @@ endforeach() install(FILES ${CSHARP_csharp_interop_BINARY} DESTINATION lib64 ) + +add_custom_target(clean_csharp + COMMENT "Removing SWIG generated C# files in ${CMAKE_CURRENT_BINARY_DIR}" + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}) + +set_target_properties(clean_csharp PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) diff --git a/src/ext/swig/arrays/arrays_csharp_impl.i b/src/ext/swig/arrays/arrays_csharp_impl.i index 120d00fb4..4831dbd28 100644 --- a/src/ext/swig/arrays/arrays_csharp_impl.i +++ b/src/ext/swig/arrays/arrays_csharp_impl.i @@ -126,3 +126,4 @@ CSHARP_ARRAYS_FIXED2(float, float) + diff --git a/src/ext/swig/arrays/arrays_impl.i b/src/ext/swig/arrays/arrays_impl.i index 2d826718a..97c1f1d7b 100644 --- a/src/ext/swig/arrays/arrays_impl.i +++ b/src/ext/swig/arrays/arrays_impl.i @@ -1,6 +1,18 @@ #if defined(SWIGCSHARP) %include "arrays_csharp_impl.i" +#elif defined(SWIGJAVA) + +%include "arrays_java.i" + +%apply signed char[] { uint8_t *buffer};// Allow unsigned char to be wrapped as a byte +%apply (char *STRING, size_t LENGTH) { (uint8_t *buffer, size_t buffer_size) }; +%apply int[] {uint16_t *} +%apply long[] {uint32_t *} +%apply long[] {uint64_t *} + +%apply float[] {float *}; + #elif defined(SWIGPYTHON) %include "arrays_numpy_impl.i" #endif diff --git a/src/ext/swig/comm.i b/src/ext/swig/comm.i new file mode 100644 index 000000000..baec17fb1 --- /dev/null +++ b/src/ext/swig/comm.i @@ -0,0 +1,54 @@ +/** Plotting model and logic + */ + +%include +%include +%include +%include "util/operator_overload.i" + +////////////////////////////////////////////// +// Don't wrap it, just use it with %import +////////////////////////////////////////////// +%import "src/ext/swig/exceptions/exceptions_impl.i" +%include "src/ext/swig/arrays/arrays_impl.i" +%import "src/ext/swig/run.i" +%import "src/ext/swig/metrics.i" + +// Ensure all the modules import the shared namespace +%pragma(csharp) moduleimports=%{ +using Illumina.InterOp.Metrics; +using Illumina.InterOp.Run; +%} + +// Ensure each of the generated C# class import the shared code +%typemap(csimports) SWIGTYPE %{ +using System; +using System.Runtime.InteropServices; +using Illumina.InterOp.Metrics; +using Illumina.InterOp.Run; +%} + + +// This imports the metrics +WRAP_METRICS(IMPORT_METRIC_WRAPPER) +// This allows exceptions to be imported, but not belong to the module +EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) + +%{ +#include "interop/io/metric_file_stream.h" +%} + + +%include "interop/io/metric_file_stream.h" + + +%define WRAP_METRIC_IO(metric_t) + %template(compute_buffer_size ) illumina::interop::io::compute_buffer_size< metric_base::metric_set >; + %template(write_interop_to_buffer ) illumina::interop::io::write_interop_to_buffer< metric_base::metric_set >; + %template(read_interop_from_buffer ) illumina::interop::io::read_interop_from_buffer< metric_base::metric_set >; + %template(read_interop ) illumina::interop::io::read_interop< metric_base::metric_set >; +%enddef + + +WRAP_METRICS(WRAP_METRIC_IO) + diff --git a/src/ext/swig/exceptions/exceptions_impl.i b/src/ext/swig/exceptions/exceptions_impl.i index 33a015c55..f061bebd6 100644 --- a/src/ext/swig/exceptions/exceptions_impl.i +++ b/src/ext/swig/exceptions/exceptions_impl.i @@ -1,4 +1,5 @@ %include "std_except.i" +%include "exception.i" #if defined(SWIGCSHARP) namespace std { @@ -9,6 +10,33 @@ namespace std %ignore invalid_argument; struct invalid_argument {}; } +#elif defined(SWIGJAVA) +%typemap(javabase) std::runtime_error "java.lang.Exception"; +%typemap(javacode) std::runtime_error %{ + public String getMessage() { + return what(); + } +%} +%typemap(javabase) std::out_of_range "java.lang.Exception"; +%typemap(javacode) std::out_of_range %{ + public String getMessage() { + return what(); + } +%} +namespace std { + struct runtime_error + { + runtime_error(const string& msg); + virtual ~runtime_error() throw(); + virtual const char* what() const throw(); + }; + struct out_of_range + { + out_of_range(const string& msg); + virtual ~out_of_range() throw(); + virtual const char* what() const throw(); + }; +} #endif %{ #include "interop/io/stream_exceptions.h" @@ -32,6 +60,29 @@ namespace std } %enddef +#elif defined(SWIGJAVA) + + %define WRAP_EXCEPTION_IMPORT(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_CSHARP) + %enddef + + %define WRAP_EXCEPTION(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_JAVA) + %typemap(throws, throws="EXCEPTION_CPLUS_PLUS") NAMESPACE EXCEPTION_CPLUS_PLUS { + // the following namespace thing is hacky + jclass excep = jenv->FindClass("com/illumina/interop/EXCEPTION_CPLUS_PLUS"); + if (excep) + jenv->ThrowNew(excep, $1.what()); + return $null; + } + %enddef + +#else + + %define WRAP_EXCEPTION_IMPORT(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_CSHARP) + %enddef + + %define WRAP_EXCEPTION(NAMESPACE, EXCEPTION_CPLUS_PLUS, EXCEPTION_CSHARP) + %enddef + #endif %define EXCEPTION_WRAPPER(WRAPPER) @@ -47,6 +98,7 @@ WRAPPER(illumina::interop::model::, invalid_channel_exception, invalid_channel_e WRAPPER(illumina::interop::model::, invalid_read_exception, invalid_read_exception) WRAPPER(illumina::interop::model::, invalid_metric_type, invalid_metric_type) WRAPPER(illumina::interop::model::, invalid_filter_option, invalid_filter_option) +WRAPPER(illumina::interop::model::, invalid_run_info_exception, invalid_run_info_exception) // XML WRAPPER(illumina::interop::xml::, xml_file_not_found_exception, xml_file_not_found_exception) diff --git a/src/ext/swig/extends/extends_csharp.i b/src/ext/swig/extends/extends_csharp.i index 45c348b71..2b53da871 100644 --- a/src/ext/swig/extends/extends_csharp.i +++ b/src/ext/swig/extends/extends_csharp.i @@ -7,7 +7,7 @@ public byte Version { get{ return (byte)version(); } } public global::System.Int64 GetKey(int lane, int tile) { - return (global::System.Int64)metric_t.id((ulong)lane, (ulong)tile); + return (global::System.Int64)metric_t.create_id((ulong)lane, (ulong)tile); } public bool HasData { get { return size() > 0; } } public metric_t GetMetric(long key) @@ -97,7 +97,7 @@ public int MaxCycle { get{ return (int)max_cycle(); } } public global::System.Int64 GetKey(int lane, int tile, int cycle) { - return (global::System.Int64)metric_t.id((ulong)lane, (ulong)tile, (ulong)cycle); + return (global::System.Int64)metric_t.create_id((ulong)lane, (ulong)tile, (ulong)cycle); } public metric_t GetMetric(int lane, int tile, int cycle) { diff --git a/src/ext/swig/extends/extends_impl.i b/src/ext/swig/extends/extends_impl.i index 50a2e299a..507055500 100644 --- a/src/ext/swig/extends/extends_impl.i +++ b/src/ext/swig/extends/extends_impl.i @@ -6,4 +6,8 @@ %enddef %define EXTEND_CYCLE_METRIC_SET(metric_t) %enddef -#endif \ No newline at end of file + %define EXTEND_Q_METRIC(metric_t) + %enddef + %define EXTEND_TILE_METRIC(metric_t) + %enddef +#endif diff --git a/src/ext/swig/imaging.i b/src/ext/swig/imaging.i index 50eadd8dc..1be0305eb 100644 --- a/src/ext/swig/imaging.i +++ b/src/ext/swig/imaging.i @@ -9,7 +9,8 @@ ////////////////////////////////////////////// // Don't wrap it, just use it with %import ////////////////////////////////////////////// -%import "src/ext/swig/exceptions/exceptions_impl.i" +%import "src/ext/swig/exceptions/exceptions_impl.i" // Import is used when the file wraps code avoiding duplicates +%include "src/ext/swig/arrays/arrays_impl.i" %import "src/ext/swig/run.i" %import "src/ext/swig/metrics.i" @@ -33,30 +34,28 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) // This allows exceptions to be imported, but not belong to the module EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) - -%template(ushort_vector) std::vector< uint16_t >; +#if defined(HAVE_UINT64_SIZE_T) && defined(SWIGPYTHON) // Python workaround + %template(map_id_offset) std::map; +#else + %template(map_id_offset) std::map; +#endif //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Imaging model //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// %{ -#include "interop/model/table/imaging_table.h" -#include "interop/logic/table/populate_imaging_table.h" +#include "interop/logic/table/create_imaging_table_columns.h" +#include "interop/logic/table/create_imaging_table.h" %} -%template(size_vector) std::vector< size_t >; -%template(size_vector_2d) std::vector< std::vector< size_t > >; -WRAP_VECTOR(illumina::interop::model::table::imaging_table); - -%include "interop/model/table/column_header.h" -%include "interop/model/table/imaging_table_entry.h" +%include "interop/model/table/imaging_column.h" %include "interop/model/table/imaging_table.h" - -%template(imaging_table_entry_vector) std::vector; -%template(column_header_vector) std::vector; +%template(imaging_column_vector) std::vector< illumina::interop::model::table::imaging_column >; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Imaging Logic //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -%include "interop/logic/table/populate_imaging_table.h" +%include "interop/logic/table/create_imaging_table.h" +%include "interop/logic/table/create_imaging_table_columns.h" + diff --git a/src/ext/swig/metrics.i b/src/ext/swig/metrics.i index 02ec1dadf..69baeeca8 100644 --- a/src/ext/swig/metrics.i +++ b/src/ext/swig/metrics.i @@ -3,6 +3,7 @@ %include %include +%include %import "src/ext/swig/exceptions/exceptions_impl.i" %include "src/ext/swig/extends/extends_impl.i" %include "src/ext/swig/arrays/arrays_impl.i" @@ -30,13 +31,21 @@ using Illumina.InterOp.Run; EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Ignore methods +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +%ignore confusion_matrix; +%ignore focus_score_matrix; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Wrap the base metrics //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// %{ #include "interop/util/time.h" -#include "interop/io/metric_file_stream.h" %} %ignore metric_group_iuo; @@ -49,7 +58,6 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %include "interop/model/metric_base/base_cycle_metric.h" %include "interop/model/metric_base/base_read_metric.h" %include "interop/model/metric_base/metric_set.h" -%include "interop/io/metric_file_stream.h" //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Define shared macros @@ -61,7 +69,6 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %apply size_t { std::map< std::size_t, metric_t >::size_type }; %apply uint64_t { metric_base::metric_set::id_t }; - %apply uint64_t { io::layout::base_metric::id_t }; WRAP_VECTOR(illumina::interop::model::metric_base::metric_set); %enddef @@ -72,10 +79,6 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %template(base_ ## metric_t ## s) metric_base::metric_set; %template(vector_ ## metric_t ## s) std::vector; - %template(compute_buffer_size ) illumina::interop::io::compute_buffer_size< metric_base::metric_set >; - %template(write_interop_to_buffer ) illumina::interop::io::write_interop_to_buffer< metric_base::metric_set >; - %template(read_interop_from_buffer ) illumina::interop::io::read_interop_from_buffer< metric_base::metric_set >; - %template(read_interop ) illumina::interop::io::read_interop< metric_base::metric_set >; %enddef @@ -94,7 +97,6 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) // List metrics //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - %define WRAP_METRICS(WRAPPER) WRAPPER(corrected_intensity_metric) WRAPPER(error_metric) @@ -107,6 +109,7 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) WRAPPER(q_by_lane_metric) %enddef + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Import metrics //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -118,12 +121,13 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// %template(index_info_vector) std::vector< illumina::interop::model::metrics::index_info >; -//%template(string_vector) std::vector< std::string >; %template(ulong_vector) std::vector< uint64_t >; %template(ushort_vector) std::vector< uint16_t >; %template(uint_vector) std::vector< uint32_t >; %template(float_vector) std::vector< float >; %template(bool_vector) std::vector< bool >; +%template(tile_metric_map) std::map< uint64_t, illumina::interop::model::metric_base::base_metric >; +%template(cycle_metric_map) std::map< uint64_t, illumina::interop::model::metric_base::base_cycle_metric >; %template(read_metric_vector) std::vector< illumina::interop::model::metrics::read_metric >; %template(q_score_bin_vector) std::vector< illumina::interop::model::metrics::q_score_bin >; @@ -163,10 +167,13 @@ WRAP_METRICS(WRAP_METRIC_SET) #include "interop/logic/metric/extraction_metric.h" #include "interop/logic/metric/q_metric.h" #include "interop/model/run_metrics.h" +#include "interop/logic/utils/metric_type_ext.h" %} %include "interop/logic/metric/extraction_metric.h" %include "interop/logic/metric/q_metric.h" %include "interop/model/run_metrics.h" +%include "interop/logic/utils/metric_type_ext.h" + diff --git a/src/ext/swig/plot.i b/src/ext/swig/plot.i index ca21528db..b03be969a 100644 --- a/src/ext/swig/plot.i +++ b/src/ext/swig/plot.i @@ -10,6 +10,7 @@ // Don't wrap it, just use it with %import ////////////////////////////////////////////// %import "src/ext/swig/exceptions/exceptions_impl.i" +%include "src/ext/swig/arrays/arrays_impl.i" %import "src/ext/swig/run.i" %import "src/ext/swig/metrics.i" @@ -85,6 +86,8 @@ WRAP_VECTOR(illumina::interop::model::plot::plot_data); %template(candle_stick_plot_data) illumina::interop::model::plot::plot_data; %template(bar_plot_data) illumina::interop::model::plot::plot_data; +%template(metric_type_vector) std::vector< illumina::interop::constants::metric_type>; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -105,7 +108,3 @@ WRAP_VECTOR(illumina::interop::model::plot::plot_data; -%template(plot_candle_stick_by_lane) illumina::interop::logic::plot::plot_by_lane; -%template(plot_qscore_histogram) illumina::interop::logic::plot::plot_qscore_histogram; diff --git a/src/ext/swig/run.i b/src/ext/swig/run.i index 087fc1e0e..55d4a2700 100644 --- a/src/ext/swig/run.i +++ b/src/ext/swig/run.i @@ -15,6 +15,7 @@ #include "interop/model/run/image_dimensions.h" #include "interop/model/run/info.h" #include "interop/model/run/parameters.h" +#include "interop/logic/metric/q_metric.h" %} %template(string_vector) std::vector< std::string >; diff --git a/src/ext/swig/summary.i b/src/ext/swig/summary.i index 7e56f689a..2db597679 100644 --- a/src/ext/swig/summary.i +++ b/src/ext/swig/summary.i @@ -43,6 +43,7 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) #include "interop/model/summary/metric_stat.h" #include "interop/model/summary/read_summary.h" #include "interop/model/summary/run_summary.h" +#include "interop/logic/metric/q_metric.h" %} WRAP_VECTOR(illumina::interop::model::summary::read_summary); diff --git a/src/ext/swig/util/operator_overload.i b/src/ext/swig/util/operator_overload.i index 9fd145d9b..4878e4583 100644 --- a/src/ext/swig/util/operator_overload.i +++ b/src/ext/swig/util/operator_overload.i @@ -1,5 +1,27 @@ -#if defined(SWIGCSHARP) - //%rename(ValueAt) operator[]const; +// Suppress warnings and expose operators to client languages +// Most languages do no support operator overloading like C++, so these need to be renamed or ignored. + +#if defined(SWIGCSHARP) || defined(SWIGJAVA) + +// TODO: Add python section and rename to appropriate python operator + %rename(equals) operator==; + + %rename(subtract) operator-; + %rename(add) *::operator+=; + %rename(is_less) operator< const; + %rename(copy) operator=; #else // Do nothing #endif + +%ignore operator<<; +%ignore operator>>; +%ignore *::operator() const; + +// TODO: Remove at in C++ and rename operator[] to at +%ignore *::operator[]; + + + + + diff --git a/src/interop/CMakeLists.txt b/src/interop/CMakeLists.txt index f20b49632..f79588061 100644 --- a/src/interop/CMakeLists.txt +++ b/src/interop/CMakeLists.txt @@ -1,5 +1,7 @@ set(SRCS + logic/metric/q_metric.cpp + logic/metric/extraction_metric.cpp model/run/info.cpp model/run/parameters.cpp model/metrics/corrected_intensity_metric.cpp @@ -10,7 +12,21 @@ set(SRCS model/metrics/q_metric.cpp model/metrics/index_metric.cpp model/metrics/q_collapsed_metric.cpp - model/summary/run_summary.cpp) + model/summary/run_summary.cpp + logic/plot/plot_by_cycle.cpp + logic/plot/plot_by_lane.cpp + logic/plot/plot_flowcell_map.cpp + logic/plot/plot_qscore_heatmap.cpp + logic/plot/plot_sample_qc.cpp + logic/plot/plot_qscore_histogram.cpp + model/run_metrics.cpp + logic/summary/run_summary.cpp + logic/summary/index_summary.cpp + logic/table/create_imaging_table_columns.cpp + logic/table/create_imaging_table.cpp + util/time.cpp + util/filesystem.cpp + ) set(HEADERS ../../interop/model/summary/cycle_state_summary.h @@ -72,7 +88,6 @@ set(HEADERS ../../interop/util/type_traits.h ../../interop/util/linear_hierarchy.h ../../interop/util/object_list.h - ../../interop/util/exception_specification.h ../../interop/logic/summary/map_cycle_to_read.h ../../interop/logic/summary/cycle_state_summary.h ../../interop/util/math.h @@ -103,7 +118,6 @@ set(HEADERS ../../interop/model/summary/index_flowcell_summary.h ../../interop/model/summary/index_count_summary.h ../../interop/logic/summary/index_summary.h - ../../interop/logic/table/populate_imaging_table.h ../../interop/model/table/imaging_table.h ../../interop/util/string.h ../../interop/model/metrics/q_collapsed_metric.h @@ -113,13 +127,17 @@ set(HEADERS ../../interop/logic/metric/tile_metric.h ../../interop/logic/logic.h ../../interop/interop.h - ../../interop/model/table/table_util.h - ../../interop/interop.h ../../interop/util/exception.h ../../interop/io/table/csv_format.h - ../../interop/model/table/column_header.h - ../../interop/model/table/imaging_table_entry.h - ../../interop/util/option_parser.h ../../interop/logic/metric/extraction_metric.h) + ../../interop/util/option_parser.h + ../../interop/logic/metric/extraction_metric.h + ../../interop/logic/table/create_imaging_table.h + ../../interop/model/table/imaging_column.h + ../../interop/logic/table/create_imaging_table_columns.h + ../../interop/logic/table/table_util.h + ../../interop/io/table/imaging_table_csv.h + ../../interop/logic/table/check_imaging_table_column.h + ../../interop/logic/table/table_populator.h) set(INTEROP_HEADERS ${HEADERS} PARENT_SCOPE) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" OR CMAKE_COMPILER_IS_GNUCC) @@ -133,11 +151,11 @@ check_function_exists("strptime" HAVE_STRPTIME) if(HAVE_STRPTIME) add_definitions(-DHAVE_STRPTIME) elseif(MSVC) - add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS) endif() add_library(${INTEROP_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS}) -add_library(${INTEROP_DL_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS}) +add_library(${INTEROP_DL_LIB} ${LIBRARY_TYPE} ${SRCS} ${HEADERS} ) add_dependencies(${INTEROP_LIB} version) add_dependencies(${INTEROP_DL_LIB} version) diff --git a/src/interop/logic/metric/extraction_metric.cpp b/src/interop/logic/metric/extraction_metric.cpp new file mode 100644 index 000000000..4e894e131 --- /dev/null +++ b/src/interop/logic/metric/extraction_metric.cpp @@ -0,0 +1,28 @@ +/** Logic for extraction metrics + * + * @file + * @date 7/7/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/metric/extraction_metric.h" + +namespace illumina { namespace interop { namespace logic { namespace metric +{ + + void copy_focus(const model::metric_base::metric_set& metrics, + float *focus_scores, + const size_t channel, + const size_t n) throw(model::invalid_parameter, model::index_out_of_bounds_exception) + { + typedef model::metric_base::metric_set::const_iterator const_iterator; + if(metrics.size()==0)return; + if(n < metrics.size()) throw model::invalid_parameter("Buffer size too small for metric set"); + if(channel >= metrics.metrics()[0].channel_count()) + throw model::invalid_parameter("Channel exceeds channel count"); + for (const_iterator it = metrics.begin(); it != metrics.end(); ++it, ++focus_scores) + *focus_scores = it->focus_score(channel); + } + + +}}}} diff --git a/src/interop/logic/metric/q_metric.cpp b/src/interop/logic/metric/q_metric.cpp new file mode 100644 index 000000000..afe9633ae --- /dev/null +++ b/src/interop/logic/metric/q_metric.cpp @@ -0,0 +1,268 @@ +/** Logic for q-metrics + * + * @file + * @date 3/12/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include "interop/logic/metric/q_metric.h" + + +namespace illumina { namespace interop { namespace logic { namespace metric +{ + + /** Populate cumulative q-metric distribution + * + * @param metric_set q-metric set + */ + template + void populate_cumulative_distribution_t(model::metric_base::metric_set& metric_set) + throw( model::index_out_of_bounds_exception ) + { + if(metric_set.size()==0) return; + typedef QMetric q_metric; + typedef model::metric_base::metric_set q_metric_set; + typedef typename q_metric_set::id_vector id_vector; + typedef typename q_metric_set::uint_t uint_t; + typedef typename id_vector::const_iterator const_id_iterator; + id_vector lane_ids = metric_set.lanes(); + for(const_id_iterator lane_beg = lane_ids.begin(), lane_end = lane_ids.end();lane_beg != lane_end;++lane_beg) + { + const uint_t lane_id = *lane_beg; + id_vector tile_ids = metric_set.tile_numbers_for_lane(lane_id); + for(const_id_iterator tile_beg = tile_ids.begin(), tile_end = tile_ids.end();tile_beg != tile_end;++tile_beg) + { + const uint_t tile_id = *tile_beg; + size_t prev_idx = metric_set.find(lane_id, tile_id, 1); + if(prev_idx >= metric_set.size()) continue; + QMetric& metric = metric_set.at(prev_idx); + metric.accumulate(metric); + const uint_t second_cycle_start = 2; // We have to accumulate the first cycle with itself, and every + // subsequent with the previous cycle. + // Also this is not 0-indexed, so we start with 2, the 2nd cycle + for(uint_t cycle = second_cycle_start;cycle <= metric_set.max_cycle();++cycle) + { + const size_t cur_idx = metric_set.find(lane_id, tile_id, cycle); + if(cur_idx>=metric_set.size() || prev_idx>=metric_set.size()) + continue;// TODO: if this happens zero out following q-scores + metric_set.at(cur_idx).accumulate(metric_set.at(prev_idx)); + prev_idx=cur_idx; + } + } + } + } + /** Populate cumulative by lane q-metric distribution + * + * @param q_metric_set q-metric set + */ + void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) + throw( model::index_out_of_bounds_exception ) + { + if(q_metric_set.size()==0) return; + typedef model::metrics::q_by_lane_metric q_by_lane_metric; + typedef model::metric_base::metric_set q_by_lane_metric_set; + typedef q_by_lane_metric_set::id_vector id_vector; + typedef q_by_lane_metric_set::uint_t uint_t; + id_vector lane_ids = q_metric_set.lanes(); + const size_t tile_id = 0; + for(id_vector::const_iterator lane_beg = lane_ids.begin(), lane_end = lane_ids.end();lane_beg != lane_end;++lane_beg) + { + size_t prev_idx = q_metric_set.find(*lane_beg, tile_id, 1); + if(prev_idx >= q_metric_set.size()) continue; + q_by_lane_metric& metric = q_metric_set.at(prev_idx); + metric.accumulate(metric); + const uint_t second_cycle_start = 2; // We have to accumulate the first cycle with itself, and every + // subsequent with the previous cycle. + // Also this is not 0-indexed, so we start with 2, the 2nd cycle + for(uint_t cycle = second_cycle_start;cycle <= q_metric_set.max_cycle();++cycle) + { + const size_t cur_idx = q_metric_set.find(*lane_beg, tile_id, cycle); + if(cur_idx>=q_metric_set.size() || prev_idx>=q_metric_set.size()) + continue;// TODO: if this happens zero out following q-scores + q_metric_set.at(cur_idx).accumulate(q_metric_set.at(prev_idx)); + prev_idx=cur_idx; + } + } + } + /** Populate cumulative q-metric distribution + * + * @note This can exist here or in SWIG. This is a swig interface function. + * @param q_metric_set q-metric set + */ + void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) + throw( model::index_out_of_bounds_exception ) + { + populate_cumulative_distribution_t(q_metric_set); + } + /** Populate cumulative cpllapsed q-metric distribution + * + * @note This can exist here or in SWIG. This is a swig interface function. + * @param q_metric_set q-metric set + */ + void populate_cumulative_distribution(model::metric_base::metric_set& q_metric_set) + throw( model::index_out_of_bounds_exception ) + { + populate_cumulative_distribution_t(q_metric_set); + } + /** Count number of unique counts to determine number + * of unique bins for legacy binning + * + * @note, if the number of bins is greater than 7, than this function stops counting! + * + * @param q_metric_set q-metric set + * @return number of unique bins + */ + size_t count_legacy_q_score_bins(const model::metric_base::metric_set& q_metric_set) + { + // 0 is a sentinel that indicates legacy binning is not required + if(q_metric_set.version() > 4) return 0; // Version 5 and later do not require legacy binning + if(!q_metric_set.bins().empty()) return 0; // If the metrics already have a header they do not require binning + + const size_t max_bin_count = 7; + model::metric_base::metric_set::const_iterator beg = q_metric_set.begin(), + end=q_metric_set.end(); + if(beg == end) return 0 ; + typedef model::metrics::q_metric::uint_t uint_t; + std::set bins_found; + for(;beg != end;++beg) + { + for(uint_t i=0;i(beg->qscore_hist().size());++i) + if(beg->qscore_hist()[i] > 0) bins_found.insert(i); + if(bins_found.size() > max_bin_count) break; // Number of bins greater than 7 indicates this is unbinned + } + return bins_found.size(); + } + /** Populate the q-score header bins from the data + * + * This only for legacy platforms that use older q-metric formats, which do not include bin information + * in the header. + * + * @param q_score_bins vector of q-score bins + * @param instrument instrument type + * @param count number of bins + */ + void populate_legacy_q_score_bins(std::vector& q_score_bins, + const constants::instrument_type instrument, + const size_t count) + { + typedef model::metrics::q_score_bin q_score_bin; + if(!requires_legacy_bins(count)) return; + q_score_bins.reserve(count); + if(instrument == constants::NextSeq) + { + q_score_bins.push_back(q_score_bin(0, 9, 8)); + q_score_bins.push_back(q_score_bin(10, 19, 13)); + q_score_bins.push_back(q_score_bin(20, 24, 22)); + q_score_bins.push_back(q_score_bin(25, 29, 27)); + q_score_bins.push_back(q_score_bin(30, 34, 32)); + q_score_bins.push_back(q_score_bin(35, 39, 37)); + } + else if(count == 7) + { + q_score_bins.push_back(q_score_bin(0, 9, 6)); + q_score_bins.push_back(q_score_bin(10, 19, 15)); + q_score_bins.push_back(q_score_bin(20, 24, 22)); + q_score_bins.push_back(q_score_bin(25, 29, 27)); + q_score_bins.push_back(q_score_bin(30, 34, 33)); + q_score_bins.push_back(q_score_bin(35, 39, 37)); + q_score_bins.push_back(q_score_bin(40, 49, 40)); + } + else if(count == 6) + { + q_score_bins.push_back(q_score_bin(0, 9, 7)); + q_score_bins.push_back(q_score_bin(10, 19, 16)); + q_score_bins.push_back(q_score_bin(20, 26, 24)); + q_score_bins.push_back(q_score_bin(27, 29, 29)); + q_score_bins.push_back(q_score_bin(30, 34, 33)); + q_score_bins.push_back(q_score_bin(35, 49, 38)); + } + else if(count == 5) + { + q_score_bins.push_back(q_score_bin(0, 9, 7)); + q_score_bins.push_back(q_score_bin(10, 19, 16)); + q_score_bins.push_back(q_score_bin(20, 29, 25)); + q_score_bins.push_back(q_score_bin(30, 34, 33)); + q_score_bins.push_back(q_score_bin(35, 49, 38)); + } + else if(count == 4) + { + q_score_bins.push_back(q_score_bin(0, 9, 7)); + q_score_bins.push_back(q_score_bin(10, 29, 20)); + q_score_bins.push_back(q_score_bin(30, 34, 33)); + q_score_bins.push_back(q_score_bin(35, 49, 38)); + } + else if(count == 3) + { + q_score_bins.push_back(q_score_bin(0, 9, 7)); + q_score_bins.push_back(q_score_bin(10, 29, 20)); + q_score_bins.push_back(q_score_bin(30, 49, 36)); + } + else if(count == 2) + { + q_score_bins.push_back(q_score_bin(0, 27, 13)); + q_score_bins.push_back(q_score_bin(28, 49, 35)); + } + else + { + q_score_bins.push_back(q_score_bin(0, 50, 20)); + } + } + /** Generate collapsed Q-metric data from Q-metrics + * + * @param metric_set q-metric set + * @param collapsed collapsed Q-metrics + */ + void create_collapse_q_metrics(const model::metric_base::metric_set& metric_set, + model::metric_base::metric_set& collapsed) + { + typedef model::metric_base::metric_set::const_iterator const_iterator; + typedef model::metric_base::metric_set::uint_t uint_t; + + const uint_t q20_idx = static_cast(index_for_q_value(metric_set, 20)); + const uint_t q30_idx = static_cast(index_for_q_value(metric_set, 30)); + + collapsed.set_version(metric_set.version()); + for(const_iterator beg = metric_set.begin(), end = metric_set.end();beg != end;++beg) + { + const uint_t q20 = beg->total_over_qscore(q20_idx); + const uint_t q30 = beg->total_over_qscore(q30_idx); + const uint_t total = beg->sum_qscore(); + const uint_t median = beg->median(metric_set.bins()); + collapsed.insert(model::metrics::q_collapsed_metric(beg->lane(), + beg->tile(), + beg->cycle(), + q20, + q30, + total, + median)); + } + } + + /** Generate by lane Q-metric data from Q-metrics + * + * @param metric_set Q-metrics + * @param bylane bylane Q-metrics + * @throws index_out_of_bounds_exception + */ + void create_q_metrics_by_lane(const model::metric_base::metric_set& metric_set, + model::metric_base::metric_set& bylane) + throw(model::index_out_of_bounds_exception) + { + typedef model::metric_base::metric_set::const_iterator const_iterator; + typedef model::metric_base::metric_set::header_type header_type; + typedef model::metric_base::base_cycle_metric::id_t id_t; + + bylane = static_cast(metric_set); + bylane.set_version(metric_set.version()); + for(const_iterator beg = metric_set.begin(), end = metric_set.end();beg != end;++beg) + { + const id_t id = model::metric_base::base_cycle_metric::create_id(beg->lane(), 0, beg->cycle()); + if(bylane.has_metric(id)) + bylane.get_metric_ref(id).accumulate_by_lane(*beg); + else + bylane.insert(model::metrics::q_by_lane_metric(beg->lane(), 0, beg->cycle(), beg->qscore_hist())); + } + } + +}}}} diff --git a/src/interop/logic/plot/plot_by_cycle.cpp b/src/interop/logic/plot/plot_by_cycle.cpp new file mode 100644 index 000000000..b2dbaea9a --- /dev/null +++ b/src/interop/logic/plot/plot_by_cycle.cpp @@ -0,0 +1,385 @@ +/** Plot an arbitrary cycle metric by cycle + * + * @file + * @date 4/28/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/plot/plot_by_cycle.h" +#include "interop/logic/metric/metric_value.h" +#include "interop/logic/plot/plot_point.h" +#include "interop/logic/plot/plot_data.h" +#include "interop/logic/metric/q_metric.h" + +namespace illumina { namespace interop { namespace logic { namespace plot +{ + + /** Plot the average over all tiles of a specific metric by cycle + * + * @param metrics set of metric records + * @param proxy functor that takes a metric record and returns a metric value + * @param options filter for metric records + * @param type type of metric to extract using the proxy functor + * @param points collection of points where x is cycle number and y is the mean metric value + */ + template + size_t populate_metric_average_by_cycle(const MetricSet& metrics, + MetricProxy& proxy, + const model::plot::filter_options& options, + const constants::metric_type type, + model::plot::data_point_collection& points) + { + const size_t max_cycle = metrics.max_cycle(); + points.assign(max_cycle, Point()); + const float dummy_x = 1; + for(typename MetricSet::const_iterator b = metrics.begin(), e = metrics.end();b != e;++b) + { + if(!options.valid_tile(*b)) continue; + const float val = proxy(*b, type); + if(std::isnan(val)) continue; + points[b->cycle()-1].add(dummy_x, val); + } + size_t index = 0; + for(size_t cycle=0;cycle(points[cycle].x()) == 0) continue; + const float avg = points[cycle].y()/points[cycle].x(); + points[index].set(static_cast(cycle+1), avg); + ++index; + } + points.resize(index); + return max_cycle; + } + /** Plot the candle stick over all tiles of a specific metric by cycle + * + * @param metrics set of metric records + * @param proxy functor that takes a metric record and returns a metric value + * @param options filter for metric records + * @param type type of metric to extract using the proxy functor + * @param points collection of points where x is cycle number and y is the candle stick metric values + * @return last populated cycle + */ + template + size_t populate_candle_stick_by_cycle(const MetricSet& metrics, + MetricProxy& proxy, + const model::plot::filter_options& options, + const constants::metric_type type, + model::plot::data_point_collection& points) + { + const size_t max_cycle = metrics.max_cycle(); + const size_t tile_count = static_cast(std::ceil(static_cast(metrics.size())/max_cycle)); + std::vector< std::vector > tile_by_cycle(max_cycle); + for(size_t i=0;i outliers; + outliers.reserve(10); // TODO: use as flag for keeping outliers + + for(typename MetricSet::const_iterator b = metrics.begin(), e = metrics.end();b != e;++b) + { + if(!options.valid_tile(*b)) continue; + const float val = proxy(*b, type); + if(std::isnan(val)) continue; + tile_by_cycle[b->cycle()-1].push_back(val); + } + points.resize(max_cycle); + size_t j=0; + for(size_t cycle=0;cycle(cycle+1), + outliers); + ++j; + } + points.resize(j); + return max_cycle; + } + /** Generate meta data for multiple plot series that compare data by channel + * + * @param channels collection of channel names + * @param data plot data that holds a collection of plot series + */ + template + void setup_series_by_channel(const std::vector& channels, model::plot::plot_data& data) + { + typedef model::plot::series series_t; + data.resize(channels.size()); + std::vector expected_order(channels.size()); + utils::actual2expected(channels, expected_order); + for(size_t i=0;i(expected_order[i])), + series_t::Line); + } + } + /** Generate meta data for multiple plot series that compare data by base + * + * @param data plot data that holds a collection of plot series + */ + template + void setup_series_by_base(model::plot::plot_data& data) + { + typedef model::plot::series series_t; + data.resize(constants::NUM_OF_BASES); + for(size_t i=0;i(i)), + constants::to_string(static_cast(i)), + series_t::Line); + } + } + /** Plot a specified metric value by cycle + * + * @ingroup plot_logic + * @param metrics run metrics + * @param type specific metric value to plot by cycle + * @param options options to filter the data + * @param data output plot data + */ + template + void plot_by_cycle_t(model::metrics::run_metrics& metrics, + const constants::metric_type type, + const model::plot::filter_options& options, + model::plot::plot_data& data) + { + data.clear(); + if(!options.all_cycles()) + INTEROP_THROW(model::invalid_filter_option, "Filtering by cycle is not supported"); + if(!options.all_reads()) + INTEROP_THROW(model::invalid_filter_option, "Filtering by read is not supported"); + if(!utils::is_cycle_metric(type)) + INTEROP_THROW(model::invalid_filter_option, "Only cycle metrics are supported"); + options.validate(type, metrics.run_info()); + size_t max_cycle=0; + switch(logic::utils::to_group(type)) + { + case constants::Extraction: + { + if(options.all_channels(type)) + { + setup_series_by_channel(metrics.run_info().channels(), data); + for(size_t i=0;i proxy(i); + max_cycle = populate_metric_average_by_cycle( + metrics.get_set(), + proxy, + options, + type, + data[i] + ); + } + } + else + { + data.assign(1, model::plot::series()); + const size_t channel = options.channel(); + metric::metric_value proxy(channel); + max_cycle = populate_candle_stick_by_cycle( + metrics.get_set(), + proxy, + options, + type, + data[0]); + } + break; + } + case constants::CorrectedInt: + { + if(options.all_bases(type)) + { + setup_series_by_base(data); + for(size_t i=0;i proxy( + static_cast(i)); + max_cycle = populate_metric_average_by_cycle( + metrics.get_set(), + proxy, + options, + type, + data[i] + ); + } + } + else + { + data.assign(1, model::plot::series()); + const constants::dna_bases base = options.dna_base(); + metric::metric_value proxy(base); + max_cycle = populate_candle_stick_by_cycle( + metrics.get_set(), + proxy, + options, + type, + data[0]); + } + break; + } + case constants::Q: + { + data.assign(1, model::plot::series()); + + typedef model::metrics::q_collapsed_metric metric_t; + metric::metric_value proxy2; + if(0 == metrics.get_set().size()) + logic::metric::create_collapse_q_metrics(metrics.get_set(), + metrics.get_set()); + max_cycle = populate_candle_stick_by_cycle( + metrics.get_set(), + proxy2, + options, + type, + data[0]); + break; + } + case constants::Error://TODO: skip last cycle of read for error metric + { + data.assign(1, model::plot::series()); + metric::metric_value proxy3; + max_cycle = populate_candle_stick_by_cycle( + metrics.get_set(), + proxy3, + options, + type, + data[0]); + break; + } + default: + INTEROP_THROW(model::invalid_metric_type, "Invalid metric group"); + } + if(type != constants::FWHM) + { + auto_scale_y(data); + if(type == constants::ErrorRate) + { + data.set_yrange(0, std::min(5.0f, data.y_axis().max())); + } + else if(type == constants::BasePercent) + { + data.set_yrange(0, std::max(50.0f, data.y_axis().max())); + } + } + else data.set_yrange(0, 6); + data.set_xrange(0, static_cast(max_cycle+2)); + data.set_xlabel("Cycle"); + data.set_ylabel(utils::to_description(type)); + + std::string title = metrics.run_info().flowcell().barcode(); + if(title != "") title += " "; + title += options.lane_description(); + if(logic::utils::is_channel_metric(type)) + title += " " + options.channel_description(metrics.run_info().channels()); + if(logic::utils::is_base_metric(type)) + title += " " + options.base_description(); + if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) + title += " " + options.surface_description(); + data.set_title(title); + } + + /** Plot a specified metric value by cycle + * + * @ingroup plot_logic + * @todo Is this temporary? + * @param metrics run metrics + * @param metric_name name of metric value to plot by cycle + * @param options options to filter the data + * @param data output plot data + */ + template + void plot_by_cycle_t(model::metrics::run_metrics& metrics, + const std::string& metric_name, + const model::plot::filter_options& options, + model::plot::plot_data& data) + { + const constants::metric_type type = constants::parse(metric_name); + if(type == constants::UnknownMetricType) + INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); + plot_by_cycle_t(metrics, type, options, data); + } + + /** Plot a specified metric value by cycle + * + * @ingroup plot_logic + * @param metrics run metrics + * @param type specific metric value to plot by cycle + * @param options options to filter the data + * @param data output plot data + */ + void plot_by_cycle(model::metrics::run_metrics& metrics, + const constants::metric_type type, + const model::plot::filter_options& options, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_metric_type, + model::invalid_channel_exception, + model::invalid_filter_option, + model::invalid_read_exception) + { + plot_by_cycle_t(metrics, type, options, data); + } + + /** Plot a specified metric value by cycle using the candle stick model + * + * @ingroup plot_logic + * @todo Is this temporary? + * @param metrics run metrics + * @param metric_name name of metric value to plot by cycle + * @param options options to filter the data + * @param data output plot data + */ + void plot_by_cycle(model::metrics::run_metrics& metrics, + const std::string& metric_name, + const model::plot::filter_options& options, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_filter_option, + model::invalid_channel_exception, + model::invalid_metric_type) + { + plot_by_cycle_t(metrics, metric_name, options, data); + } + + /** List metric types available for by cycle plots + * + * @param types destination vector to fill with metric types + * @param ignore_accumulated if true, ignore accumulated Q20 and Q30 + */ + void list_by_cycle_metrics(std::vector& types, const bool ignore_accumulated) + { + types.clear(); + std::vector tmp; + constants::list_enums(tmp); + types.reserve(tmp.size()); + for(size_t i=0;i& names, const bool ignore_accumulated) + { + std::vector types; + list_by_cycle_metrics(types, ignore_accumulated); + names.clear(); + names.reserve(types.size()); + for(size_t i=0;i + void populate_candle_stick_by_lane(const MetricSet& metrics, + MetricProxy& proxy, + const model::plot::filter_options& options, + const constants::metric_type type, + model::plot::data_point_collection& points) + { + if(metrics.max_lane() == 0) return; + const size_t lane_count = metrics.max_lane(); + const size_t tile_count = static_cast(std::ceil(static_cast(metrics.size())/lane_count)); + std::vector< std::vector > tile_by_lane(metrics.max_lane()); + for(size_t i=0;i outliers; + outliers.reserve(10); + + for(typename MetricSet::const_iterator b = metrics.begin(), e = metrics.end();b != e;++b) + { + if(!options.valid_tile(*b)) continue; + const float val = proxy(*b, type); + if(std::isnan(val)) continue; + tile_by_lane[b->lane()-1].push_back(val); + } + points.resize(lane_count); + size_t offset=0; + for(size_t i=0;i(i+1); + plot_candle_stick(points[offset], tile_by_lane[i].begin(), tile_by_lane[i].end(), lane, outliers); + ++offset; + } + points.resize(offset); + } + + /** Plot a specified metric value by lane + * + * @ingroup plot_logic + * @param metrics run metrics + * @param type specific metric value to plot by lane + * @param options options to filter the data + * @param data output plot data + */ + template + void plot_by_lane_t(const model::metrics::run_metrics& metrics, + const constants::metric_type type, + const model::plot::filter_options& options, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_metric_type, + model::invalid_filter_option) + { + if(utils::is_cycle_metric(type)) + INTEROP_THROW(model::invalid_metric_type, "Cycle metrics are unsupported"); + options.validate(type, metrics.run_info()); + data.assign(1, model::plot::series(utils::to_description(type), "Blue")); + metric::metric_value proxy3(options.read()); + populate_candle_stick_by_lane(metrics.get_set(), proxy3, options, type, + data[0]); + + const size_t read_count = metrics.run_info().reads().size(); + if(utils::is_read_metric(type) && options.all_reads() && read_count > 1) + INTEROP_THROW(model::invalid_filter_option, "All reads is unsupported for run with " << read_count); + + if(type == constants::ClusterCount || type == constants::Clusters )//constants::Density ) + { + data.push_back(model::plot::series("PF", "DarkGreen")); + const constants::metric_type second_type = + (type==constants::Clusters ? constants::ClustersPF : constants::ClusterCountPF); + //(type==constants::Density ? constants::DensityPF : constants::ClusterCountPF); + populate_candle_stick_by_lane(metrics.get_set(), proxy3, options, second_type, + data[1]); + } + + auto_scale(data, true, 1.2f); + if(type == constants::PercentPrephasing || type == constants::PercentPhasing) + data.set_yrange(0, 1); + data.set_xrange(0, data.x_axis().max()+1); + + data.set_xlabel("Lane"); + data.set_ylabel(utils::to_description(type)); + + std::string title = metrics.run_info().flowcell().barcode(); + if(options.is_specific_read(type)) + { + if(title != "") title += " "; + title += options.read_description(); + } + if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) + { + if(title != "") title += " "; + title += options.surface_description(); + } + data.set_title(title); + } + /** Plot a specified metric value by lane + * + * @ingroup plot_logic + * @param metrics run metrics + * @param type specific metric value to plot by lane + * @param options options to filter the data + * @param data output plot data + */ + void plot_by_lane(const model::metrics::run_metrics& metrics, + const constants::metric_type type, + const model::plot::filter_options& options, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_metric_type, + model::invalid_filter_option) + { + plot_by_lane_t(metrics, type, options, data); + } + + /** Plot a specified metric value by cycle + * + * @ingroup plot_logic + * @todo Is this temporary? + * @param metrics run metrics + * @param metric_name name of metric value to plot by cycle + * @param options options to filter the data + * @param data output plot data + */ + void plot_by_lane(const model::metrics::run_metrics& metrics, + const std::string& metric_name, + const model::plot::filter_options& options, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception, + model::invalid_metric_type, + model::invalid_filter_option) + { + const constants::metric_type type = constants::parse(metric_name); + if(type == constants::UnknownMetricType) + INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); + plot_by_lane(metrics, type, options, data); + } + + /** List metric types available for by lane plots + * + * @param types destination vector to fill with metric types + * @param ignore_pf if true, ignore density PF and cluster PF + */ + void list_by_lane_metrics(std::vector& types, const bool ignore_pf) + { + std::vector tmp; + constants::list_enums(tmp); + types.clear(); + types.reserve(tmp.size()); + for(size_t i=0;i& names, const bool ignore_pf) + { + std::vector types; + list_by_lane_metrics(types, ignore_pf); + + names.clear(); + names.reserve(types.size()); + for(size_t i=0;i + void populate_flowcell_map(I beg, + I end, + MetricProxy& proxy, + const constants::metric_type type, + const model::run::flowcell_layout& layout, + const model::plot::filter_options &options, + model::plot::flowcell_data& data, + std::vector& values_for_scaling) + { + if(beg == end) return; + const bool all_surfaces = !options.is_specific_surface(); + for (;beg != end;++beg) + { + if( !options.valid_tile_cycle(*beg) ) continue; + const float val = proxy(*beg, type); + if(std::isnan(val)) continue; + data.set_data(beg->lane()-1, + beg->physical_location_index( + layout.naming_method(), + layout.sections_per_lane(), + layout.tile_count(), + layout.swath_count(), + all_surfaces), + beg->tile(), + val); + values_for_scaling.push_back(val); + } + } + + /** Plot a flowcell map + * + * @ingroup plot_logic + * @param metrics run metrics + * @param type specific metric value to plot by cycle + * @param options options to filter the data + * @param data output flowcell map + * @param buffer preallocated memory for data + * @param tile_buffer preallocated memory for tile ids + */ + void plot_flowcell_map(model::metrics::run_metrics& metrics, + const constants::metric_type type, + const model::plot::filter_options& options, + model::plot::flowcell_data& data, + float* buffer, + ::uint32_t* tile_buffer) + throw(model::invalid_filter_option, + model::invalid_metric_type, + model::index_out_of_bounds_exception) + { + const model::run::flowcell_layout& layout = metrics.run_info().flowcell(); + data.clear(); + if(buffer == 0 || tile_buffer==0) + data.resize(layout.lane_count(), + layout.total_swaths(layout.surface_count() > 1 && !options.is_specific_surface()), + layout.tiles_per_lane()); + else + data.set_buffer(buffer, tile_buffer, layout.lane_count(), + layout.total_swaths(layout.surface_count() > 1 && !options.is_specific_surface()), + layout.tiles_per_lane()); + std::vector values_for_scaling; + values_for_scaling.reserve(data.length()); + + options.validate(type, metrics.run_info()); + + if(utils::is_cycle_metric(type) && options.all_cycles()) + INTEROP_THROW( model::invalid_filter_option, "All cycles is unsupported"); + if(utils::is_read_metric(type) && options.all_reads() && metrics.run_info().reads().size() > 1) + INTEROP_THROW( model::invalid_filter_option, "All reads is unsupported"); + switch(logic::utils::to_group(type)) + { + case constants::Tile: + { + typedef model::metrics::tile_metric metric_t; + typedef model::metric_base::metric_set metric_set_t; + const metric_set_t& metric_set = metrics.get_set(); + metric::metric_value proxy(options.read()); + populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, + values_for_scaling); + break; + } + case constants::Extraction: + { + typedef model::metrics::extraction_metric metric_t; + typedef model::metric_base::metric_set metric_set_t; + const metric_set_t& metric_set = metrics.get_set(); + const size_t channel = options.channel(); + if(options.all_channels(type)) + INTEROP_THROW(model::invalid_filter_option, "All channels is unsupported"); + metric::metric_value proxy(channel); + populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, + values_for_scaling); + break; + } + case constants::CorrectedInt: + { + typedef model::metrics::corrected_intensity_metric metric_t; + typedef model::metric_base::metric_set metric_set_t; + const metric_set_t& metric_set = metrics.get_set(); + const constants::dna_bases base = options.dna_base(); + if(options.all_bases(type)) + INTEROP_THROW( model::invalid_filter_option, "All bases is unsupported"); + metric::metric_value proxy(base); + populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, + values_for_scaling); + break; + } + case constants::Q: + { + typedef model::metrics::q_collapsed_metric metric_t; + typedef model::metric_base::metric_set metric_set_t; + metric_set_t &metric_set = metrics.get_set(); + if(0 == metric_set.size()) + { + logic::metric::create_collapse_q_metrics(metrics.get_set(), metric_set); + if(type == constants::AccumPercentQ20 || type == constants::AccumPercentQ30) + logic::metric::populate_cumulative_distribution(metric_set); + } + metric::metric_value proxy; + populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, + values_for_scaling); + break; + } + case constants::Error: + { + typedef model::metrics::error_metric metric_t; + typedef model::metric_base::metric_set metric_set_t; + const metric_set_t& metric_set = metrics.get_set(); + metric::metric_value proxy; + populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, + values_for_scaling); + break; + } + default: + INTEROP_THROW( model::invalid_metric_type, "Unsupported metric type: " << constants::to_string(type)); + }; + + if(!values_for_scaling.empty()) + { + std::sort(values_for_scaling.begin(), values_for_scaling.end()); + // TODO: Put this back + /* + const float lower = util::percentile_sorted(values_for_scaling.begin(), values_for_scaling.end(), + 25); + const float upper = util::percentile_sorted(values_for_scaling.begin(), values_for_scaling.end(), + 75);*/ + const float lower = values_for_scaling[size_t(0.25*values_for_scaling.size())]; + const float upper = values_for_scaling[size_t(0.75*values_for_scaling.size())]; + data.set_range(std::max(lower - 2 * (upper - lower), values_for_scaling[0]), + std::min(values_for_scaling.back(), upper + 2 * (upper - lower))); + } + else data.set_range(0,0); + if(type == constants::ErrorRate) data.set_range(0, std::min(5.0f, data.saxis().max())); + + std::string title = metrics.run_info().flowcell().barcode(); + if(title != "") title += " "; + title += utils::to_description(type); + data.set_title(title); + + std::string subtitle; + if(metrics.run_info().flowcell().surface_count()>1) + subtitle += options.surface_description() + " "; + subtitle += options.cycle_description(); + if(logic::utils::is_channel_metric(type)) + subtitle += " " + options.channel_description(metrics.run_info().channels()); + if(logic::utils::is_base_metric(type)) + subtitle += " " + options.base_description(); + if(logic::utils::is_read_metric(type)) + subtitle += " " + options.read_description(); + data.set_subtitle(subtitle); + data.set_label(utils::to_description(type)); + } + /** Plot a flowcell map + * + * @ingroup plot_logic + * @param metrics run metrics + * @param metric_name specific metric value to plot by cycle + * @param options options to filter the data + * @param data output flowcell map + * @param buffer preallocated memory for data + * @param tile_buffer preallocated memory for tile ids + */ + void plot_flowcell_map(model::metrics::run_metrics& metrics, + const std::string& metric_name, + const model::plot::filter_options& options, + model::plot::flowcell_data& data, + float* buffer, + ::uint32_t* tile_buffer) + throw(model::invalid_filter_option, + model::invalid_metric_type, + model::index_out_of_bounds_exception) + { + const constants::metric_type type = constants::parse(metric_name); + if(type == constants::UnknownMetricType) + INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); + plot_flowcell_map(metrics, type, options, data, buffer, tile_buffer); + } + /** List metric type names available for flowcell + * + * @param types destination vector to fill with metric type names + */ + void list_flowcell_metrics(std::vector& types) + { + types.clear(); + std::vector tmp; + constants::list_enums(tmp); + types.reserve(tmp.size()); + for(size_t i=0;i& names) + { + std::vector types; + list_flowcell_metrics(types); + names.clear(); + names.reserve(types.size()); + for(size_t i=0;i 1 && !options.is_specific_surface())) * + layout.tiles_per_lane(); + } + + +}}}} diff --git a/src/interop/logic/plot/plot_qscore_heatmap.cpp b/src/interop/logic/plot/plot_qscore_heatmap.cpp new file mode 100644 index 000000000..b02eccdb1 --- /dev/null +++ b/src/interop/logic/plot/plot_qscore_heatmap.cpp @@ -0,0 +1,203 @@ +/** Plot the Q-score heat map + * + * @file + * @date 5/5/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/plot/plot_qscore_heatmap.h" + +#include "interop/model/plot/bar_point.h" +#include "interop/logic/metric/q_metric.h" + +namespace illumina { namespace interop { namespace logic { namespace plot +{ + + + /** Populate the q-score heat map based on the filter options + * + * @param beg iterator to start of q-metric collection + * @param end iterator to end of q-metric collection + * @param bins q-score bins + * @param options filter for metric records + * @param data q-score heatmap + */ + template + void populate_heatmap_from_compressed(I beg, + I end, + const std::vector& bins, + const model::plot::filter_options &options, + model::plot::heatmap_data& data) + { + for (;beg != end;++beg) + { + if( !options.valid_tile(*beg) ) continue; + for(size_t bin =0;bin < bins.size();++bin) + data(beg->cycle()-1, bins[bin].value()-1) += beg->qscore_hist(bin); + } + } + /** Populate the q-score heatmap based on the filter options + * + * @param beg iterator to start of q-metric collection + * @param end iterator to end of q-metric collection + * @param options filter for metric records + * @param data q-score heatmap + */ + template + void populate_heatmap_from_uncompressed(I beg, + I end, + const model::plot::filter_options &options, + model::plot::heatmap_data& data) + { + for (;beg != end;++beg) + { + if( !options.valid_tile(*beg) ) continue; + + for(size_t bin =0;bin < beg->size();++bin) + data(beg->cycle()-1, bin) += beg->qscore_hist(bin); + } + } + /** Normalize the heat map to a percent + * + * @param data output heat map data + */ + void normalize_heatmap(model::plot::heatmap_data& data) + { + float max_value = 0; + for(size_t r=0;r + void remap_to_bins(I beg, I end, const size_t max_cycle, model::plot::heatmap_data& data) + { + for(;beg != end;++beg) + { + for(size_t bin = std::max(0, beg->lower()-1), upper=beg->upper();bin < upper;++bin) + { + for(size_t cycle = 0;cycle < max_cycle;++cycle) + { + data(cycle, bin) = data(cycle, beg->value()-1); + } + } + } + } + /** Plot a heat map of q-scores + * + * @param metric_set q-metrics (full or by lane) + * @param options options to filter the data + * @param data output heat map data + * @param buffer preallocated memory + */ + template + void populate_heatmap(const model::metric_base::metric_set& metric_set, + const model::plot::filter_options& options, + model::plot::heatmap_data& data, + float* buffer) + { + const size_t max_q_val = logic::metric::max_qval(metric_set); + const size_t max_cycle = metric_set.max_cycle(); + if(buffer != 0) data.set_buffer(buffer, max_cycle, max_q_val); + else data.resize(max_cycle, max_q_val); + INTEROP_ASSERT(data.row_count() > 0); + INTEROP_ASSERTMSG(data.column_count() > 0, data.column_count() << ", " << metric_set.size() << ", " + << metric_set.bin_count() << ", " + << metric::is_compressed(metric_set) << ", " + << metric_set.bins().back().upper()); + const bool is_compressed = logic::metric::is_compressed(metric_set); + if(is_compressed) + populate_heatmap_from_compressed(metric_set.begin(), + metric_set.end(), + metric_set.bins(), + options, + data); + else + populate_heatmap_from_uncompressed(metric_set.begin(), + metric_set.end(), + options, + data); + normalize_heatmap(data); + remap_to_bins(metric_set.bins().begin(), + metric_set.bins().end(), + data.row_count(), + data); + } + /** Plot a heat map of q-scores + * + * @ingroup plot_logic + * @param metrics run metrics + * @param options options to filter the data + * @param data output heat map data + * @param buffer preallocated memory + */ + void plot_qscore_heatmap(model::metrics::run_metrics& metrics, + const model::plot::filter_options& options, + model::plot::heatmap_data& data, + float* buffer) + throw(model::index_out_of_bounds_exception, + model::invalid_filter_option) + { + options.validate(constants::QScore, metrics.run_info()); + data.clear(); + if(options.is_specific_surface()) + { + typedef model::metrics::q_metric metric_t; + if (metrics.get_set().size() == 0)return; + populate_heatmap(metrics.get_set(), options, data, buffer); + } + else + { + typedef model::metrics::q_by_lane_metric metric_t; + if(0 == metrics.get_set().size()) + logic::metric::create_q_metrics_by_lane(metrics.get_set(), + metrics.get_set()); + if (metrics.get_set().size() == 0)return; + populate_heatmap(metrics.get_set(), options, data, buffer); + } + + data.set_xrange(0, static_cast(data.row_count())); + data.set_yrange(0, static_cast(data.column_count())); + + data.set_xlabel("Cycle"); + data.set_ylabel("Q Score"); + + std::string title = metrics.run_info().flowcell().barcode(); + if(title != "") title += " "; + title += options.lane_description(); + if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) + title += " " + options.surface_description(); + data.set_title(title); + } + /** Count number of rows for the heat map + * + * @param metrics run metrics + * @return number of rows + */ + size_t count_rows_for_heatmap(const model::metrics::run_metrics& metrics) + { + return metrics.get_set< model::metrics::q_metric >().max_cycle(); + } + /** Count number of columns for the heat map + * + * @param metrics run metrics + * @return number of columns + */ + size_t count_columns_for_heatmap(const model::metrics::run_metrics& metrics) + { + return logic::metric::max_qval(metrics.get_set< model::metrics::q_metric >()); + } + + + +}}}} diff --git a/src/interop/logic/plot/plot_qscore_histogram.cpp b/src/interop/logic/plot/plot_qscore_histogram.cpp new file mode 100644 index 000000000..641ed2d78 --- /dev/null +++ b/src/interop/logic/plot/plot_qscore_histogram.cpp @@ -0,0 +1,274 @@ +/** Plot the Q-score histogram + * + * @file + * @date 5/5/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/plot/plot_qscore_histogram.h" +#include "interop/logic/metric/q_metric.h" + +namespace illumina { namespace interop { namespace logic { namespace plot +{ + + + /** Populate the q-score histogram based on the filter options + * + * @param beg iterator to start of q-metric collection + * @param end iterator to end of q-metric collection + * @param options filter for metric records + * @param first_cycle first cycle to keep + * @param last_cycle last cycle to keep + * @param histogram q-score histogram + */ + template + void populate_distribution(I beg, + I end, + const model::plot::filter_options &options, + const size_t first_cycle, + const size_t last_cycle, + std::vector& histogram) + { + if(beg == end) return; + histogram.resize(beg->size(), 0); + for (;beg != end;++beg) + { + if( !options.valid_tile(*beg) || beg->cycle() < first_cycle || beg->cycle() > last_cycle) continue; + beg->accumulate_into(histogram); + } + } + /** Scale the histogram if necessary and provide the scale label + * + * @param histogram q-score histogram + * @return label of the scale + */ + inline std::string scale_histogram(std::vector& histogram) + { + float max_height = 0; + for(size_t i=0;i(options.cycle())); + return last_cycle; + } + /** Plot an unbinned histogram + * + * @param histogram q-score histogram + * @param points collection of points where x is lane number and y is the candle stick metric values + * @return max x-value + */ + template + float plot_unbinned_histogram(const std::vector& histogram, + model::plot::data_point_collection& points) + { + points.resize(histogram.size()); + size_t point_index = 0; + for(size_t i=0;i(i+1), histogram[i]); + ++point_index; + } + points.resize(point_index); + if(point_index == 0) return 0; + return static_cast(points[point_index-1].x()+1); + } + /** Plot a binned histogram + * + * @param beg iterator to start of the histogram bins + * @param end iterator to end of the histogram bins + * @param histogram q-score histogram + * @param points collection of points where x is lane number and y is the candle stick metric values + * @return max x-value + */ + template + float plot_binned_histogram(I beg, + I end, + const std::vector& histogram, + model::plot::data_point_collection& points) + { + float max_x_value = 0; + points.resize(std::distance(beg, end)); + size_t point_index = 0; + if(points.size() == histogram.size()) //Compressed + { + for (size_t i=0; beg != end; ++beg, ++i) + { + INTEROP_ASSERT(i < histogram.size()); + if(histogram[i] == 0)continue; + points[point_index].set(beg->lower(), histogram[i], static_cast(beg->upper()-beg->lower()+1)); + max_x_value = std::max(max_x_value, points[point_index].x()+points[point_index].width()); + ++point_index; + } + } + else // Uncompressed + { + for (size_t i=0; beg != end; ++beg, ++i) + { + const size_t bin = static_cast(beg->value())-1; + INTEROP_ASSERTMSG(bin < histogram.size(), bin << " < " << histogram.size()); + if(histogram[bin] == 0)continue; + points[point_index].set(beg->lower(), histogram[bin], static_cast(beg->upper()-beg->lower()+1)); + max_x_value = std::max(max_x_value, points[point_index].x()+points[point_index].width()); + ++point_index; + } + } + points.resize(point_index); + return max_x_value; + } + + /** Plot a histogram of q-scores + * + * @ingroup plot_logic + * @param metrics run metrics + * @param options options to filter the data + * @param data output plot data + * @param boundary index of bin to create the boundary sub plots (0 means do nothing) + */ + void plot_qscore_histogram(model::metrics::run_metrics& metrics, + const model::plot::filter_options& options, + model::plot::plot_data& data, + const size_t boundary) + throw( model::invalid_read_exception, + model::index_out_of_bounds_exception, + model::invalid_filter_option) + { + typedef model::plot::bar_point Point; + options.validate(constants::QScore, metrics.run_info()); + data.clear(); + const size_t first_cycle = options.all_reads() ? 1 : metrics.run_info().read(options.read()).first_cycle(); + + data.push_back(model::plot::series("Q Score", "Blue", model::plot::series::Bar)); + if(boundary>0) + { + data.push_back(model::plot::series("Q Score", "DarkGreen", model::plot::series::Bar)); + data.push_back(model::plot::series("Threshold(>=Q30)", "Green", model::plot::series::Line)); + } + + //"DarkGreen" + + for(size_t i=0;i histogram; + float max_x_value; + + if(options.is_specific_surface()) + { + typedef model::metrics::q_metric metric_t; + const size_t last_cycle = get_last_filtered_cycle(metrics.run_info(), + options, + metrics.get_set().max_cycle()); + if(metrics.get_set().size() == 0) + { + //data.clear(); // TODO: Add this back? + return; + } + populate_distribution( + metrics.get_set().begin(), + metrics.get_set().end(), + options, + first_cycle, + last_cycle, + histogram); + axis_scale = scale_histogram(histogram); + if(!metrics.get_set().bins().empty()) + max_x_value=plot_binned_histogram(metrics.get_set().bins().begin(), + metrics.get_set().bins().end(), + histogram, + data[0]); + else max_x_value=plot_unbinned_histogram(histogram, data[0]); + } + else + { + typedef model::metrics::q_by_lane_metric metric_t; + if(0 == metrics.get_set().size()) + logic::metric::create_q_metrics_by_lane(metrics.get_set(), + metrics.get_set()); + if(0 == metrics.get_set().size()) + { + //data.clear(); // TODO: Add this back? + return; + } + const size_t last_cycle = get_last_filtered_cycle(metrics.run_info(), + options, + metrics.get_set().max_cycle()); + INTEROP_ASSERT(0 != metrics.get_set().size()); + populate_distribution( + metrics.get_set().begin(), + metrics.get_set().end(), + options, + first_cycle, + last_cycle, + histogram); + axis_scale = scale_histogram(histogram); + if(!metrics.get_set().bins().empty()) + max_x_value=plot_binned_histogram(metrics.get_set().bins().begin(), + metrics.get_set().bins().end(), + histogram, + data[0]); + else max_x_value=plot_unbinned_histogram(histogram, data[0]); + } + + // split into two series, < Q30 and Q30 <= + if(data.size() > 1) + { + size_t gt_eq_boundary_count = 0; + data[1].resize(data[0].size()); + for (size_t i = 0; i < data[0].size(); ++i) + { + if (data[0][i].x() >= boundary) + { + data[1][gt_eq_boundary_count] = data[0][i]; + ++gt_eq_boundary_count; + } + } + data[0].resize(data[0].size() - gt_eq_boundary_count); + data[1].resize(gt_eq_boundary_count); + } + + + auto_scale_y(data, false); + data.set_xrange(1, std::max(1.0f, max_x_value)*1.1f); + if(data.size() > 2) + { + data[2].resize(2); + data[2][0].set(static_cast(boundary), data.y_axis().min()); + data[2][1].set(static_cast(boundary), data.y_axis().max()); + } + + data.set_xlabel("Q Score"); + data.set_ylabel("Total ("+axis_scale+")"); + + std::string title = metrics.run_info().flowcell().barcode(); + if(title != "") title += " "; + title += options.lane_description(); + if(options.is_specific_read()) + title += " " + options.read_description(); + if(metrics.run_info().flowcell().surface_count()>1 && options.is_specific_surface()) + title += " " + options.surface_description(); + data.set_title(title); + } + + +}}}} diff --git a/src/interop/logic/plot/plot_sample_qc.cpp b/src/interop/logic/plot/plot_sample_qc.cpp new file mode 100644 index 000000000..9fb1a1a9f --- /dev/null +++ b/src/interop/logic/plot/plot_sample_qc.cpp @@ -0,0 +1,115 @@ +/** Plot reads identified versus index + * + * @file + * @date 5/10/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/plot/plot_sample_qc.h" + +#include "interop/util/math.h" +#include "interop/logic/utils/enums.h" + +namespace illumina { namespace interop { namespace logic { namespace plot { + + /** Populate reads identified versus the index + * + * @param index_metrics set of metric records + * @param tile_metrics source collection of tile metrics + * @param lane lane index + * @param points collection of points where x is lane number and y is the candle stick metric values + */ + template + float populate_reads_identified(const model::metric_base::metric_set& index_metrics, + const model::metric_base::metric_set& tile_metrics, + const size_t lane, + model::plot::data_point_collection& points) + { + typedef model::metric_base::metric_set index_metric_set_t; + typedef std::map index_count_map_t; + typedef typename index_count_map_t::iterator map_iterator; + typedef typename index_count_map_t::const_iterator const_map_iterator; + typedef typename model::metrics::index_metric::const_iterator const_index_iterator; + + index_count_map_t index_count_map; + float pf_cluster_count_total = 0; + for(typename index_metric_set_t::const_iterator b = index_metrics.begin(), e = index_metrics.end();b != e;++b) + { + if(lane != b->lane()) continue; + try + { + const model::metrics::tile_metric &tile_metric = tile_metrics.get_metric(b->lane(), b->tile()); + pf_cluster_count_total += tile_metric.cluster_count_pf(); + for(const_index_iterator ib = b->indices().begin(), ie = b->indices().end();ib != ie;++ib) + { + map_iterator found_index = index_count_map.find(ib->index_seq()); + if(found_index == index_count_map.end()) index_count_map[ib->index_seq()] = ib->count(); + else found_index->second += ib->count(); + } + } + catch(const model::index_out_of_bounds_exception&){continue;} // TODO: check better? + } + points.resize(index_count_map.size()); + float max_height=0; + std::vector heights; + heights.reserve(index_count_map.size()); + size_t i=0; + std::vector keys; + keys.reserve(index_count_map.size()); + for(const_map_iterator b = index_count_map.begin(), e = index_count_map.end();b != e;++b) + keys.push_back(b->first); + std::stable_sort(keys.begin(), keys.end()); + for(typename std::vector::const_iterator b = keys.begin(), e = keys.end();b != e;++b,++i) + { + const float height = (pf_cluster_count_total==0) ? 0 : index_count_map[*b] / pf_cluster_count_total * 100.0f; + points[i].set(i+1.0f, height, 1.0f); + max_height = std::max(max_height, height); + } + return max_height; + } + + /** Plot reads identified versus index + * + * @ingroup plot_logic + * @param metrics run metrics + * @param lane lane number + * @param data output plot data + */ + void plot_sample_qc(const model::metrics::run_metrics& metrics, + const size_t lane, + model::plot::plot_data& data) + throw(model::index_out_of_bounds_exception) + { + typedef model::plot::series bar_series_t; + data.set_xlabel("Index Number"); + data.set_ylabel("% Reads Identified (PF)"); + data.assign(1, bar_series_t("% reads", "Green", bar_series_t::Bar)); + data[0].add_option(constants::to_string(constants::Centered)); + + if(metrics.get_set().size() == 0) + { + if(metrics.run_info().is_indexed()) + { + data.set_range(data.x_axis().min(), + 1, + data.y_axis().min(), + 5); + } + //data.clear(); // TODO: Remove below and uncomment this line + + return; + } + + const float max_height = populate_reads_identified(metrics.get_set(), + metrics.get_set(), + lane, + data[0]); + auto_scale(data); + data.set_range(data.x_axis().min(), + static_cast(data[0].size()+1), + data.y_axis().min(), + roundf(max_height+5)); + } + + +}}}} diff --git a/src/interop/logic/summary/index_summary.cpp b/src/interop/logic/summary/index_summary.cpp new file mode 100644 index 000000000..71dbccc86 --- /dev/null +++ b/src/interop/logic/summary/index_summary.cpp @@ -0,0 +1,164 @@ +/** Summary logic for index metrics + * + * @file + * @date 5/11/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/summary/index_summary.h" + +#include "interop/util/statistics.h" + + +namespace illumina { namespace interop { namespace logic { namespace summary { + + /** Summarize a index metrics for a specific lane + * + * @param beg iterator to start of source collection of index metrics + * @param end iterator to end of source collection of index metrics + * @param tile_metrics source collection of tile metrics + * @param lane lane number + * @param summary destination index flowcell summary + */ + template + void summarize_index_metrics(I beg, + I end, + const model::metric_base::metric_set& tile_metrics, + const size_t lane, + model::summary::index_lane_summary &summary) + throw(model::index_out_of_bounds_exception) + { + typedef typename model::summary::index_lane_summary::read_count_t read_count_t; + typedef typename model::metrics::index_metric::const_iterator const_index_iterator; + typedef model::summary::index_count_summary index_count_summary; + typedef std::map index_count_map_t; + typedef typename index_count_map_t::iterator map_iterator; + + index_count_map_t index_count_map; + size_t total_mapped_reads = 0; + read_count_t pf_cluster_count_total = 0; + read_count_t cluster_count_total = 0; + for(;beg != end;++beg) + { + if(beg->lane() != lane) continue; + try + { + const model::metrics::tile_metric &tile_metric = tile_metrics.get_metric(beg->lane(), beg->tile()); + pf_cluster_count_total += static_cast(tile_metric.cluster_count_pf()); + cluster_count_total += static_cast(tile_metric.cluster_count()); + + for(const_index_iterator ib = beg->indices().begin(), ie = beg->indices().end();ib != ie;++ib) + { + map_iterator found_index = index_count_map.find(ib->index_seq()); + if(found_index == index_count_map.end()) + { + index_count_map[ib->index_seq()] = index_count_summary(index_count_map.size()+1,// TODO: get correspondance with plot + ib->index1(), + ib->index2(), + ib->sample_id(), + ib->sample_proj(), + ib->count()); + } + else + { + found_index->second += ib->count(); + } + total_mapped_reads += ib->count(); + } + } + catch(const model::index_out_of_bounds_exception&){continue;} // TODO: check better + } + + float max_fraction_mapped = -std::numeric_limits::max(); + float min_fraction_mapped = std::numeric_limits::max(); + summary.reserve(index_count_map.size()); + + + std::vector keys; + keys.reserve(index_count_map.size()); + for(map_iterator b = index_count_map.begin(), e = index_count_map.end();b != e;++b) + keys.push_back(b->first); + std::stable_sort(keys.begin(), keys.end()); + + for(typename std::vector::const_iterator kcurr = keys.begin(), kbeg=kcurr;kcurr != keys.end();++kcurr) + { + index_count_summary& count_summary = index_count_map[*kcurr]; + count_summary.id(static_cast(std::distance(kbeg, kcurr)+1)); + count_summary.update_fraction_mapped(static_cast(pf_cluster_count_total)); + const float fraction_mapped = count_summary.fraction_mapped(); + summary.push_back(count_summary); + max_fraction_mapped = std::max(max_fraction_mapped, fraction_mapped); + min_fraction_mapped = std::min(min_fraction_mapped, fraction_mapped); + } + + const float avg_fraction_mapped =util::mean(summary.begin(), + summary.end(), + util::op::const_member_function(&index_count_summary::fraction_mapped)); + const float std_fraction_mapped = + std::sqrt(util::variance_with_mean(summary.begin(), + summary.end(), + avg_fraction_mapped, + util::op::const_member_function(&index_count_summary::fraction_mapped))); + summary.set(total_mapped_reads, + pf_cluster_count_total, + cluster_count_total, + min_fraction_mapped, + max_fraction_mapped, + std_fraction_mapped/avg_fraction_mapped); + } + /** Summarize a collection index metrics for a specific lane + * + * @param metrics source run metrics + * @param lane lane number + * @param summary destination index lane summary + */ + void summarize_index_metrics(const model::metrics::run_metrics &metrics, + const size_t lane, + model::summary::index_lane_summary &summary) + throw(model::index_out_of_bounds_exception) + { + summarize_index_metrics(metrics.get_set().begin(), + metrics.get_set().end(), + metrics.get_set(), lane, + summary); + } + /** Summarize a collection index metrics + * + * @ingroup summary_logic + * @param index_metrics source collection of index metrics + * @param tile_metrics source collection of tile metrics + * @param lane_count number of lanes + * @param summary destination index flowcell summary + */ + void summarize_index_metrics(const model::metric_base::metric_set& index_metrics, + const model::metric_base::metric_set& tile_metrics, + const size_t lane_count, + model::summary::index_flowcell_summary &summary) + throw(model::index_out_of_bounds_exception) + { + summary.resize(lane_count); + for(size_t lane=1;lane <= lane_count;++lane) + { + summarize_index_metrics(index_metrics.begin(), index_metrics.end(), tile_metrics, lane, summary[lane-1]); + } + } + + /** Summarize index metrics from run metrics + * + * @ingroup summary_logic + * @param metrics source collection of all metrics + * @param summary destination index flowcell summary + */ + void summarize_index_metrics(const model::metrics::run_metrics &metrics, + model::summary::index_flowcell_summary &summary) + throw(model::index_out_of_bounds_exception) + { + const size_t lane_count = metrics.run_info().flowcell().lane_count(); + summarize_index_metrics(metrics.get_set(), + metrics.get_set(), + lane_count, + summary); + } + + +}}}} diff --git a/src/interop/logic/summary/run_summary.cpp b/src/interop/logic/summary/run_summary.cpp new file mode 100644 index 000000000..2f753ff82 --- /dev/null +++ b/src/interop/logic/summary/run_summary.cpp @@ -0,0 +1,141 @@ +/** Summary logic for run metrics + * + * @file + * @date 3/10/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/summary/run_summary.h" + +#include "interop/logic/summary/error_summary.h" +#include "interop/logic/summary/tile_summary.h" +#include "interop/logic/summary/extraction_summary.h" +#include "interop/logic/summary/quality_summary.h" +#include "interop/logic/utils/channel.h" +#include "interop/logic/metric/q_metric.h" + + +namespace illumina { namespace interop { namespace logic { namespace summary +{ + /** Determine maximum number of tiles among all metrics for each lane + * + * @param metrics run metrics + * @param summary run summary + */ + void summarize_tile_count(const model::metrics::run_metrics& metrics, model::summary::run_summary& summary) + { + using namespace model::metrics; + for(unsigned int lane=0;lane().tile_numbers_for_lane(lane+1).size()); + tile_count = std::max(tile_count, metrics.get_set().tile_numbers_for_lane(lane+1).size()); + tile_count = std::max(tile_count, metrics.get_set().tile_numbers_for_lane(lane+1).size()); + tile_count = std::max(tile_count, metrics.get_set().tile_numbers_for_lane(lane+1).size()); + for(size_t read=0;read 0; + } + /** Predicate to sort lane summaries by lane number + * + * @param lhs left hand side summary + * @param rhs right hand side summary + * @return true if lhs < rhs + */ + inline bool less_than(const model::summary::lane_summary& lhs, const model::summary::lane_summary& rhs) + { + return lhs.lane() < rhs.lane(); + } + } + + /** Summarize a collection run metrics + * + * TODO speed up calculation by adding no_median flag + * + * @ingroup summary_logic + * @param metrics source collection of all metrics + * @param summary destination run summary + */ + void summarize_run_metrics(model::metrics::run_metrics& metrics, model::summary::run_summary& summary) + throw( model::index_out_of_bounds_exception, + model::invalid_channel_exception ) + { + using namespace model::metrics; + if(metrics.empty()) return; + + + summary.initialize(metrics.run_info().reads(), metrics.run_info().flowcell().lane_count()); + + + read_cycle_vector_t cycle_to_read; + map_read_to_cycle_number(summary.begin(), summary.end(), cycle_to_read); + summarize_tile_metrics(metrics.get_set().begin(), metrics.get_set().end(), summary); + summarize_error_metrics(metrics.get_set().begin(), + metrics.get_set().end(), + cycle_to_read, + summary); + INTEROP_ASSERT(metrics.run_info().channels().size()>0); + const size_t intensity_channel = utils::expected2actual_map(metrics.run_info().channels())[0]; + summarize_extraction_metrics(metrics.get_set().begin(), + metrics.get_set().end(), + cycle_to_read, + intensity_channel, + summary); + + if(0 == metrics.get_set().size()) + logic::metric::create_collapse_q_metrics(metrics.get_set(), + metrics.get_set()); + summarize_collapsed_quality_metrics(metrics.get_set().begin(), + metrics.get_set().end(), + cycle_to_read, + summary); + summarize_tile_count(metrics, summary); + + summarize_cycle_state(metrics.get_set(), + metrics.get_set(), + cycle_to_read, + &model::summary::cycle_state_summary::error_cycle_range, + summary); + summarize_cycle_state(metrics.get_set(), + metrics.get_set(), + cycle_to_read, + &model::summary::cycle_state_summary::extracted_cycle_range, + summary); + summarize_cycle_state(metrics.get_set(), + metrics.get_set(), + cycle_to_read, + &model::summary::cycle_state_summary::qscored_cycle_range, + summary); + // Summarize called cycle state + summarize_cycle_state(metrics.get_set(), + metrics.get_set(), + cycle_to_read, + &model::summary::cycle_state_summary::called_cycle_range, + summary); + + // Remove the empty lane summary entries + size_t max_lane_count = 0; + for(size_t read=0;read + void populate_imaging_table_data_by_cycle(InputIterator beg, + InputIterator end, + const size_t q20_idx, + const size_t q30_idx, + const constants::tile_naming_method naming_method, + const summary::read_cycle_vector_t& cycle_to_read, + const std::vector& columns, + const std::map& row_offset, + const size_t column_count, + OutputIterator data_beg, + OutputIterator data_end) + { + for(;beg != end;++beg) + { + const Id_t id = beg->id(); + typename std::map::const_iterator row_it = row_offset.find(id); + const size_t row = row_it->second; + const summary::read_cycle& read = cycle_to_read[beg->cycle()-1]; + if(data_beg[row*column_count]==0) + { + if((beg->cycle()-1) >= cycle_to_read.size()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Cycle exceeds total cycles from Reads in the RunInfo.xml"); + + + INTEROP_ASSERTMSG(row_it != row_offset.end(), "Bug with row offset"); + INTEROP_ASSERT(row(model::table::ImagingColumnCount), columns[model::table::ReadColumn] ); + INTEROP_ASSERTMSG(columns[model::table::CycleWithinReadColumn] < static_cast(model::table::ImagingColumnCount), columns[model::table::CycleWithinReadColumn] ); + INTEROP_ASSERTMSG(data_beg+row*column_count+columns[model::table::ReadColumn] < data_end, columns[model::table::ReadColumn] + << " - " << row*column_count+columns[model::table::ReadColumn] << " < " << std::distance(data_beg, data_end) << " " + << "row: " << row << " < " << row_offset.size()); + // TODO: Only populate Id once! + table_populator::populate_id(*beg, + read, + q20_idx, + q30_idx, + naming_method, + columns, + data_beg+row*column_count, + data_end); + } + table_populator::populate(*beg, + read.number, + q20_idx, + q30_idx, + naming_method, + columns, + data_beg+row*column_count, data_end); + } + } + /** Populate the imaging table with a by cycle InterOp metric set + * + * @param metrics InterOp metric set + * @param q20_idx index of the q20 value + * @param q30_idx index of the q30 value + * @param naming_method tile naming method enum + * @param cycle_to_read map cycle to read/cycle within read + * @param columns vector of table columns + * @param row_offset offset for each metric into the sorted table + * @param column_count number of data columns including sub columns + * @param data_beg iterator to start of table data + * @param data_end iterator to end of table data + */ + template + void populate_imaging_table_data_by_cycle(const MetricSet& metrics, + const size_t q20_idx, + const size_t q30_idx, + const constants::tile_naming_method naming_method, + const summary::read_cycle_vector_t& cycle_to_read, + const std::vector& columns, + const std::map& row_offset, + const size_t column_count, + OutputIterator data_beg, OutputIterator data_end) + { + populate_imaging_table_data_by_cycle(metrics.begin(), + metrics.end(), + q20_idx, + q30_idx, + naming_method, + cycle_to_read, + columns, + row_offset, + column_count, + data_beg, data_end); + } + /** Zero out first column of every row + * + * @param beg start of collection + * @param end end of collection + * @param column_count number of columns in of collection + */ + template + void zero_first_column(I beg, I end, size_t column_count) + { + for(;beg != end;beg+=column_count) *beg = 0; + } + /** Populate the imaging table with all the metrics in the run + * + * @param metrics collection of all run metrics + * @param columns vector of table columns + * @param row_offset offset for each metric into the sorted table + * @param data_beg iterator to start of table data + * @param data_end iterator to end of table data + */ + template + void create_imaging_table_data(const model::metrics::run_metrics& metrics, + const std::vector& columns, + const std::map& row_offset, + I data_beg, + I data_end) + { + typedef typename model::metrics::run_metrics::id_t id_t; + const size_t column_count = columns.back().column_count(); + const constants::tile_naming_method naming_method = metrics.run_info().flowcell().naming_method(); + const size_t q20_idx = metric::index_for_q_value(metrics.get_set(), 20); + const size_t q30_idx = metric::index_for_q_value(metrics.get_set(), 30); + std::vector cmap(model::table::ImagingColumnCount, std::numeric_limits::max()); + for(size_t i=0;i data_end) + throw model::index_out_of_bounds_exception("Table is larger than buffer"); + zero_first_column(data_beg, data_beg+column_count*row_offset.size(), column_count); + populate_imaging_table_data_by_cycle(metrics.get_set(), + q20_idx, + q30_idx, + naming_method, + cycle_to_read, + cmap, + row_offset, + column_count, + data_beg, data_end); + populate_imaging_table_data_by_cycle(metrics.get_set(), + q20_idx, + q30_idx, + naming_method, + cycle_to_read, + cmap, + row_offset, + column_count, + data_beg, data_end); + populate_imaging_table_data_by_cycle(metrics.get_set(), + q20_idx, + q30_idx, + naming_method, + cycle_to_read, + cmap, + row_offset, + column_count, + data_beg, data_end); + populate_imaging_table_data_by_cycle(metrics.get_set(), + q20_idx, + q30_idx, + naming_method, + cycle_to_read, + cmap, + row_offset, + column_count, + data_beg, data_end); + populate_imaging_table_data_by_cycle(metrics.get_set(), + q20_idx, + q30_idx, + naming_method, + cycle_to_read, + cmap, + row_offset, + column_count, + data_beg, data_end); + const typename model::metrics::run_metrics::tile_metric_set_t& tile_metrics = + metrics.get_set(); + for(typename std::map::const_iterator it = row_offset.begin();it != row_offset.end();++it) + { + const id_t tid = model::metric_base::base_cycle_metric::tile_hash_from_id(it->first); + if (!tile_metrics.has_metric(tid)) continue; + const size_t cycle = static_cast(model::metric_base::base_cycle_metric::cycle_from_id(it->first)); + const size_t row = it->second; + const summary::read_cycle& read = cycle_to_read[cycle-1]; + table_populator::populate(tile_metrics.get_metric(tid), + read.number, + q20_idx, + q30_idx, + naming_method, + cmap, + data_beg+row*column_count, data_end); + } + } + /** Populate the imaging table with all the metrics in the run + * + * @param metrics collection of all run metrics + * @param columns vector of table columns + * @param row_offset map between metric id and row offset in table + * @param data_beg iterator to start of table data + * @param n number of cells in the data table + */ + template + void populate_imaging_table_data_t(const model::metrics::run_metrics& metrics, + const std::vector& columns, + const std::map& row_offset, + I data_beg, const size_t n) + { + create_imaging_table_data(metrics, columns, row_offset, data_beg, data_beg+n); + } + /** Populate the imaging table with all the metrics in the run + * + * @param metrics collection of all run metrics + * @param columns vector of table columns + * @param row_offset ordering for the rows + * @param data_beg iterator to start of table data + * @param n number of cells in the data table + */ + void populate_imaging_table_data(const model::metrics::run_metrics& metrics, + const std::vector& columns, + const std::map& row_offset, + float* data_beg, + const size_t n) + { + create_imaging_table_data(metrics, columns, row_offset, data_beg, data_beg+n); + } + /** Count the number of rows in the imaging table and setup an ordering + * + * @param metrics collections of InterOp metric sets + * @param row_offset ordering for the rows + */ + void count_table_rows(const model::metrics::run_metrics& metrics, + std::map& row_offset) + { + typedef model::metrics::run_metrics::cycle_metric_map_t cycle_metric_map_t; + cycle_metric_map_t hash_set; + metrics.populate_id_map(hash_set); + row_offset.clear(); + size_t row = 0; + for(cycle_metric_map_t::const_iterator it = hash_set.begin();it != hash_set.end();++it,++row) + row_offset[it->first]=row; + } + /** Count the total number of columns for the data table + * + * @param columns vector of table column descriptions + * @return total number of columns including sub columns + */ + size_t count_table_columns(const std::vector& columns) + { + if(columns.empty()) return 0; + return columns.back().column_count(); + } + /** Create an imaging table from run metrics + * + * @param metrics source run metrics + * @param table destination imaging table + */ + void create_imaging_table(const model::metrics::run_metrics& metrics, model::table::imaging_table& table) + throw(model::invalid_column_type, model::index_out_of_bounds_exception) + { + typedef model::table::imaging_table::column_vector_t column_vector_t; + typedef model::table::imaging_table::data_vector_t data_vector_t; + typedef std::map row_offset_t; + + row_offset_t row_offset; + column_vector_t columns; + create_imaging_table_columns(metrics, columns); + count_table_rows(metrics, row_offset); + data_vector_t data(row_offset.size()*count_table_columns(columns), std::numeric_limits::quiet_NaN()); + create_imaging_table_data(metrics, columns, row_offset, data.begin(), data.end()); + table.set_data(row_offset.size(), columns, data); + } + +}}}} diff --git a/src/interop/logic/table/create_imaging_table_columns.cpp b/src/interop/logic/table/create_imaging_table_columns.cpp new file mode 100644 index 000000000..3a05ebd37 --- /dev/null +++ b/src/interop/logic/table/create_imaging_table_columns.cpp @@ -0,0 +1,172 @@ +/** Logic to populate the imaging table columns + * + * @file + * @date 7/20/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/logic/table/create_imaging_table_columns.h" + + +#include "interop/logic/table/table_util.h" +#include "interop/logic/summary/map_cycle_to_read.h" +#include "interop/logic/table/check_imaging_table_column.h" +#include "interop/logic/metric/q_metric.h" + +namespace illumina { namespace interop { namespace logic { namespace table +{ + /** Determine whether a column should be filled + * + * @param metrics run metrics + * @param tile_hash map between the tile has and base metric + * @param filled destination array that indicates whether a column should be filled + */ + void determine_filled_columns(const model::metrics::run_metrics& metrics, + const model::metrics::run_metrics::tile_metric_map_t& tile_hash, + std::vector< bool >& filled) + { + typedef model::metrics::run_metrics::tile_metric_map_t tile_metric_map_t; + filled.assign(model::table::ImagingColumnCount, false); + check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); + check_imaging_table_column::set_filled_for_metric_set(metrics.get_set(), filled); + + const model::metric_base::metric_set& tile_metrics = + metrics.get_set(); + const constants::tile_naming_method naming_method = metrics.run_info().flowcell().naming_method(); + const size_t q20_idx = metric::index_for_q_value(metrics.get_set(), 20); + const size_t q30_idx = metric::index_for_q_value(metrics.get_set(), 30); + for(tile_metric_map_t::const_iterator it = tile_hash.begin();it != tile_hash.end();++it) + { + if(!tile_metrics.has_metric(it->first)) continue; + for(model::run::info::const_read_iterator read_it = metrics.run_info().reads().begin();read_it != metrics.run_info().reads().end();++read_it) + { + check_imaging_table_column::set_filled_for_metric(tile_metrics.get_metric(it->first), + read_it->number(), + q20_idx, + q30_idx, + naming_method, + filled); + } + } + const size_t column_count = std::accumulate(filled.begin(), filled.end(), static_cast(0)); + if(column_count>0) + { + check_imaging_table_column::set_id_filled(filled); + // TODO: Create is Selection Logic + filled[model::table::SectionColumn] = metrics.run_info().flowcell().naming_method() == constants::FiveDigit; + } + } + /** Convert a column type to the data type + * + * @param index imaging table column index + * @return metric data + */ + constants::metric_data to_data_type(const size_t index) + { + typedef std::pair mapped_t; +# define INTEROP_TUPLE7(Name, Ignored2, Ignored3, Ignored4, Ignored5, Data, Ignored6) mapped_t(model::table::Name##Column, constants::Data), + static const mapped_t name_types[] = {INTEROP_IMAGING_COLUMN_TYPES};// mapped_t(model::table::ImagingColumnCount, constants::MetricDataCount)}; +# undef INTEROP_TUPLE7 + return util::constant_mapping_get(name_types, static_cast(index), constants::UnknownMetricData); + } + /** Get the number of digits to round a column + * + * @param index imaging table column id + * @return number of digits to round a column + */ + size_t get_column_rounding(const model::table::column_id index) + { + typedef std::pair mapped_t; +# define INTEROP_TUPLE7(Name, Ignored2, Ignored3, Ignored4, Ignored5, Ignored6, RoundDigits) mapped_t(model::table::Name##Column, RoundDigits), + static const mapped_t name_types[] = {INTEROP_IMAGING_COLUMN_TYPES};// mapped_t(model::table::ImagingColumnCount, constants::MetricDataCount)}; +# undef INTEROP_TUPLE7 + return util::constant_mapping_get(name_types, index, static_cast(0)); + } + /** Create the imaging table columns + * + * @param channels names of each channel + * @param filled boolean array indicating whether to include the column + * @param columns destination column vector + */ + void create_imaging_table_columns(const std::vector& channels, + const std::vector& filled, + std::vector< model::table::imaging_column >& columns) + throw(model::invalid_column_type, model::index_out_of_bounds_exception) + { + columns.clear(); + columns.reserve(model::table::ImagingColumnCount+channels.size()+constants::NUM_OF_BASES); + if(filled.size() != model::table::ImagingColumnCount) + INTEROP_THROW(model::invalid_column_type, "Filled vector does not match number of column names"); + for(size_t i=0, offset=0;i tmp; + constants::list_enum_names(tmp, 1); + tmp.resize(constants::NUM_OF_BASES); + columns.push_back(model::table::imaging_column(i, offset, tmp)); + offset+=tmp.size(); + break; + } + case constants::ChannelArray: + { + columns.push_back(model::table::imaging_column(i, offset, channels)); + offset+=channels.size(); + break; + } + default: + { + INTEROP_THROW( model::invalid_column_type, "Column index does not have a type"); + } + } + } + } + /** Populate the value offsets from the column headers + * + * @param columns column headers + */ + void populate_column_offsets(std::vector& columns) + throw(model::invalid_column_type) + { + size_t offset = 0; + for(size_t i=0;i& columns) + throw(model::invalid_column_type, model::index_out_of_bounds_exception) + { + typedef model::metrics::run_metrics::tile_metric_map_t tile_metric_map_t; + std::vector< bool > filled; + tile_metric_map_t tile_hash; + metrics.populate_id_map(tile_hash); + determine_filled_columns(metrics, tile_hash, filled); + create_imaging_table_columns(metrics.run_info().channels(), filled, columns); + } + + +}}}} diff --git a/src/interop/model/metrics/corrected_intensity_metric.cpp b/src/interop/model/metrics/corrected_intensity_metric.cpp index 08166d197..4cb339578 100644 --- a/src/interop/model/metrics/corrected_intensity_metric.cpp +++ b/src/interop/model/metrics/corrected_intensity_metric.cpp @@ -3,7 +3,6 @@ * Each version of the corrected intensity metrics file has a layout defined below. * * @file - * * @date 8/6/2015 * @version 1.0 * @copyright GNU Public License. @@ -108,7 +107,7 @@ namespace illumina{ namespace interop{ namespace io { sizeof(metric_id_t)+ sizeof(intensity_t) + // m_average_cycle_intensity sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_all - sizeof(intensity_t)*constants::NUM_OF_BASES + // m_correctedIntCalled + sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_called sizeof(count_t)*constants::NUM_OF_BASES_AND_NC + // m_called_counts sizeof(float) // m_signal_to_noise ); @@ -202,7 +201,7 @@ namespace illumina{ namespace interop{ namespace io { { return static_cast( sizeof(metric_id_t)+ - sizeof(intensity_t)*constants::NUM_OF_BASES + // m_correctedIntCalled + sizeof(intensity_t)*constants::NUM_OF_BASES + // m_corrected_int_called sizeof(count_t)*constants::NUM_OF_BASES_AND_NC // m_called_counts ); } @@ -216,10 +215,7 @@ namespace illumina{ namespace interop{ namespace io { } }; #pragma pack() - - } - } -} +}}} INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(corrected_intensity_metric, 2 ) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(corrected_intensity_metric, 3 ) diff --git a/src/interop/model/metrics/error_metric.cpp b/src/interop/model/metrics/error_metric.cpp index 36e99f3d5..64e2bc525 100644 --- a/src/interop/model/metrics/error_metric.cpp +++ b/src/interop/model/metrics/error_metric.cpp @@ -3,7 +3,6 @@ * Each version of the error metrics file has a layout defined below. * * @file - * * @date 8/19/2015 * @version 1.0 * @copyright GNU Public License. @@ -108,7 +107,7 @@ namespace illumina{ namespace interop{ namespace io { #pragma pack() - }}} +}}} INTEROP_FORCE_LINK_DEF(error_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(error_metric, 3 ) diff --git a/src/interop/model/metrics/extraction_metric.cpp b/src/interop/model/metrics/extraction_metric.cpp index adfff62e6..1bd804fe9 100644 --- a/src/interop/model/metrics/extraction_metric.cpp +++ b/src/interop/model/metrics/extraction_metric.cpp @@ -3,18 +3,19 @@ * Each version of the extraction metrics file has a layout defined below. * * @file - * * @date 8/20/2015 * @version 1.0 * @copyright GNU Public License. */ #include +#include "interop/util/math.h" #include "interop/model/metrics/extraction_metric.h" #include "interop/io/format/metric_format_factory.h" using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ namespace io { +namespace illumina { namespace interop { namespace io +{ #pragma pack(1) /** Extraction Metric Record Layout Version 2 * @@ -136,7 +137,7 @@ namespace illumina{ namespace interop{ namespace io { #pragma pack() - }}} +}}} INTEROP_FORCE_LINK_DEF(extraction_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(extraction_metric, 2 ) diff --git a/src/interop/model/metrics/image_metric.cpp b/src/interop/model/metrics/image_metric.cpp index 24ef2e2b5..b257a8e7c 100644 --- a/src/interop/model/metrics/image_metric.cpp +++ b/src/interop/model/metrics/image_metric.cpp @@ -3,7 +3,6 @@ * Each version of the image metrics file has a layout defined below. * * @file - * * @date 8/20/2015 * @version 1.0 * @copyright GNU Public License. @@ -15,7 +14,8 @@ using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ namespace io { +namespace illumina { namespace interop { namespace io +{ #pragma pack(1) /** Image Metric Record Layout Version 1 * @@ -104,12 +104,12 @@ namespace illumina{ namespace interop{ namespace io { std::streamsize count = 0; metric_id_t metric_id; metric_id.set(metric); - for(image_metric::ushort_t channelIndex=0;channelIndex < image_metric::MAX_CHANNELS;++channelIndex) + for(image_metric::ushort_t channel_index=0;channel_index < image_metric::MAX_CHANNELS;++channel_index) { - if(channelIndex > 0) write_binary(stream, metric_id); - count += stream_map< channel_t >(stream, channelIndex); - count += stream_map< contrast_t >(stream, metric.m_min_contrast[channelIndex]); - count += stream_map< contrast_t >(stream, metric.m_max_contrast[channelIndex]); + if(channel_index > 0) write_binary(stream, metric_id); + count += stream_map< channel_t >(stream, channel_index); + count += stream_map< contrast_t >(stream, metric.m_min_contrast[channel_index]); + count += stream_map< contrast_t >(stream, metric.m_max_contrast[channel_index]); } return count; } @@ -212,7 +212,7 @@ namespace illumina{ namespace interop{ namespace io { */ static record_size_t compute_size(const image_metric::header_type& header) { - return static_cast(sizeof(metric_id_t)+header.channelCount()*sizeof(contrast_t)*2); + return static_cast(sizeof(metric_id_t)+header.channel_count()*sizeof(contrast_t)*2); } /** Map reading/writing a header to a stream * @@ -237,7 +237,7 @@ namespace illumina{ namespace interop{ namespace io { } }; #pragma pack() // DO NOT MOVE - }}} + }}} INTEROP_FORCE_LINK_DEF(image_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 1 ) diff --git a/src/interop/model/metrics/index_metric.cpp b/src/interop/model/metrics/index_metric.cpp index 484b284fa..450abea18 100644 --- a/src/interop/model/metrics/index_metric.cpp +++ b/src/interop/model/metrics/index_metric.cpp @@ -3,7 +3,6 @@ * Each version of the index metrics file has a layout defined below. * * @file - * * @date 11/2/15. * @version 1.0 * @copyright GNU Public License. @@ -14,149 +13,159 @@ using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ namespace io { -#pragma pack(1) -/** Index Metric Record Layout Version 1 - * - * This class provides an interface to reading the index metric file: - * - InterOp/IndexMetrics.bin - * - InterOp/IndexMetricsOut.bin - * - * The class takes two template arguments: - * - * 1. Metric Type: index_metric - * 2. Version: 1 - */ -template<> -struct generic_layout : public default_layout<1> +namespace illumina { namespace interop { namespace io { - /** @page index_v1 Index Version 1 +#pragma pack(1) + + /** Index Metric Record Layout Version 1 * * This class provides an interface to reading the index metric file: * - InterOp/IndexMetrics.bin * - InterOp/IndexMetricsOut.bin * - * The file format for index metrics is as follows: - * - * @b Header - * - * illumina::interop::io::read_metrics (Function that parses this information) - * - * byte 0: version number + * The class takes two template arguments: * - * @b n-Records - * - * illumina::interop::io::layout::base_read_metric (Class that parses this information) - * - * 2 bytes: lane number (uint16) - * 2 bytes: tile number (uint16) - * 2 bytes: read number (uint16) - * - * illumina::interop::io::generic_layout (Class that parses this information) - * - * 2 bytes: index name length (indexNameLength) (uint16) - * indexNameLength bytes: index name - * 4 bytes: index count (uint32) - * 2 bytes: sample name length (sampleNameLength) (uint16) - * sampleNameLength bytes: sample name - * 2 bytes: project name length (projectNameLength) (uint16) - * projectNameLength bytes: project name + * 1. Metric Type: index_metric + * 2. Version: 1 */ - /** Metric ID type */ - typedef layout::base_read_metric metric_id_t; - /** Defines an empty header */ - typedef void* header_t; - /** No record size is required for this stream */ - //typedef fixed_record_size record_size_t; - enum{BYTE_COUNT=1, RECORD_SIZE=sizeof(metric_id_t)+BYTE_COUNT}; - - /** Read metric from the input stream - * - * @param in input stream - * @param metric destination metric - * @return sentinel - */ - template - static std::streamsize map_stream(std::istream& in, Metric& metric, Header&, const bool) + template<> + struct generic_layout : public default_layout<1> { - std::string index_name; - ::uint32_t count; - std::string sample_name; - std::string project_name; + /** @page index_v1 Index Version 1 + * + * This class provides an interface to reading the index metric file: + * - InterOp/IndexMetrics.bin + * - InterOp/IndexMetricsOut.bin + * + * The file format for index metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number + * + * @b n-Records + * + * illumina::interop::io::layout::base_read_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 2 bytes: tile number (uint16) + * 2 bytes: read number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 2 bytes: index name length (indexNameLength) (uint16) + * indexNameLength bytes: index name + * 4 bytes: index count (uint32) + * 2 bytes: sample name length (sampleNameLength) (uint16) + * sampleNameLength bytes: sample name + * 2 bytes: project name length (projectNameLength) (uint16) + * projectNameLength bytes: project name + */ + /** Metric ID type */ + typedef layout::base_read_metric metric_id_t; + /** Defines an empty header */ + typedef void *header_t; + /** No record size is required for this stream */ + //typedef fixed_record_size record_size_t; + enum + { + BYTE_COUNT = 1, RECORD_SIZE = sizeof(metric_id_t) + BYTE_COUNT + }; - read_binary(in, index_name); - if(in.fail()) - INTEROP_THROW( incomplete_file_exception, "No more data after index name"); - read_binary(in, count); - if(in.fail()) - INTEROP_THROW(incomplete_file_exception, "No more data after count"); - read_binary(in, sample_name); - if(in.fail()) - INTEROP_THROW(incomplete_file_exception, "No more data after sample name"); - read_binary(in, project_name); - index_metric::index_array_t::iterator beg = metric.m_indices.begin(), end = metric.m_indices.end(); - for(;beg != end;++beg) if(beg->index_seq() == sample_name) break; - if( beg == end ) + /** Read metric from the input stream + * + * @param in input stream + * @param metric destination metric + * @return sentinel + */ + template + static std::streamsize map_stream(std::istream &in, Metric &metric, Header &, const bool) { - metric.m_indices.push_back(index_info(index_name, sample_name, project_name, count)); + std::string index_name; + ::uint32_t count; + std::string sample_name; + std::string project_name; + + read_binary(in, index_name); + if (in.fail()) + INTEROP_THROW(incomplete_file_exception, "No more data after index name"); + read_binary(in, count); + if (in.fail()) + INTEROP_THROW(incomplete_file_exception, "No more data after count"); + read_binary(in, sample_name); + if (in.fail()) + INTEROP_THROW(incomplete_file_exception, "No more data after sample name"); + read_binary(in, project_name); + index_metric::index_array_t::iterator beg = metric.m_indices.begin(), end = metric.m_indices.end(); + for (; beg != end; ++beg) if (beg->index_seq() == sample_name) break; + if (beg == end) + { + metric.m_indices.push_back(index_info(index_name, sample_name, project_name, count)); + } + else beg->m_count += count; + + return BYTE_COUNT; } - else beg->m_count += count; - return BYTE_COUNT; - } - /** Write metric to the output stream - * - * @param out output stream - * @param metric source metric - * @return sentinel - */ - template - static std::streamsize map_stream(std::ostream& out, Metric& metric, Header&, const bool) - { - metric_id_t metric_id; - metric_id.set(metric); - for(index_metric::index_array_t::const_iterator beg = metric.indices().begin(), - end = metric.indices().end();beg != end;++beg) + /** Write metric to the output stream + * + * @param out output stream + * @param metric source metric + * @return sentinel + */ + template + static std::streamsize map_stream(std::ostream &out, Metric &metric, Header &, const bool) { - if(beg != metric.indices().begin()) write_binary(out, metric_id); - write_binary(out, beg->index_seq()); - write_binary(out, static_cast< ::uint32_t >(beg->count())); - write_binary(out, beg->sample_id()); - write_binary(out, beg->sample_proj()); + metric_id_t metric_id; + metric_id.set(metric); + for (index_metric::index_array_t::const_iterator beg = metric.indices().begin(), + end = metric.indices().end(); beg != end; ++beg) + { + if (beg != metric.indices().begin()) write_binary(out, metric_id); + write_binary(out, beg->index_seq()); + write_binary(out, static_cast< ::uint32_t >(beg->count())); + write_binary(out, beg->sample_id()); + write_binary(out, beg->sample_proj()); + } + return BYTE_COUNT; + } + + /** Compute the layout size + * + * @note The size of the record is not known ahead of time, so we just return 1. + * + * @return sentinel + */ + static record_size_t compute_size(const index_metric::header_type &) + { + return static_cast(RECORD_SIZE); } - return BYTE_COUNT; - } - /** Compute the layout size - * - * @note The size of the record is not known ahead of time, so we just return 1. - * - * @return sentinel - */ - static record_size_t compute_size(const index_metric::header_type&) - { - return static_cast(RECORD_SIZE); - } - /** Compute header size - * - * @return header size - */ - static size_t compute_header_size(const index_metric::header_type&) - { - return sizeof(version_t); - } - /** Skip reading/writing record size to this file - * - * @return sentinel - */ - template - static record_size_t map_stream_record_size(Stream&, record_size_t) - { - return static_cast(RECORD_SIZE); - } -}; + + /** Compute header size + * + * @return header size + */ + static size_t compute_header_size(const index_metric::header_type &) + { + return sizeof(version_t); + } + + /** Skip reading/writing record size to this file + * + * @return sentinel + */ + template + static record_size_t map_stream_record_size(Stream &, record_size_t) + { + return static_cast(RECORD_SIZE); + } + }; #pragma pack() }}} INTEROP_FORCE_LINK_DEF(index_metric) -INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(index_metric, 1 ) + +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(index_metric, 1) diff --git a/src/interop/model/metrics/q_metric.cpp b/src/interop/model/metrics/q_metric.cpp index 283722c4b..f3d5ae1c0 100644 --- a/src/interop/model/metrics/q_metric.cpp +++ b/src/interop/model/metrics/q_metric.cpp @@ -3,7 +3,6 @@ * Each version of the q-score metrics file has a layout defined below. * * @file - * * @date 8/21/2015 * @version 1.0 * @copyright GNU Public License. @@ -17,7 +16,8 @@ using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ namespace io { +namespace illumina { namespace interop { namespace io +{ #pragma pack(1) /** Q-score Metric Record Layout Version 4 @@ -531,7 +531,7 @@ namespace illumina{ namespace interop{ namespace io { }; #pragma pack()// DO NOT MOVE - }}} + }}} INTEROP_FORCE_LINK_DEF(q_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_metric, 4 ) diff --git a/src/interop/model/metrics/tile_metric.cpp b/src/interop/model/metrics/tile_metric.cpp index 591a5cf1b..7d6e8fb22 100644 --- a/src/interop/model/metrics/tile_metric.cpp +++ b/src/interop/model/metrics/tile_metric.cpp @@ -3,7 +3,6 @@ * Each version of the tile metrics file has a layout defined below. * * @file - * * @date 8/6/2015 * @version 1.0 * @copyright GNU Public License. @@ -11,12 +10,12 @@ #include "interop/model/metrics/tile_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/util/math.h" using namespace illumina::interop::model::metrics; -namespace illumina{ namespace interop{ - - namespace io { +namespace illumina { namespace interop { namespace io +{ #pragma pack(1) /** Tile Metric Record Layout Version 2 * @@ -121,20 +120,20 @@ namespace illumina{ namespace interop{ if( rec.code % Phasing < 100 ) { //code = Prephasing+read*2; - int codeOffset = rec.code % Phasing; - if(codeOffset%2 == 0) + int code_offset = rec.code % Phasing; + if(code_offset%2 == 0) { - get_read(metric, (codeOffset/2)+1)->percent_phasing(val*100); + get_read(metric, (code_offset/2)+1)->percent_phasing(val*100); } else { - get_read(metric, (codeOffset+1)/2)->percent_prephasing(val*100); + get_read(metric, (code_offset+1)/2)->percent_prephasing(val*100); } } else if(rec.code % PercentAligned < 100) { - int codeOffset = rec.code % PercentAligned; - get_read(metric, codeOffset+1)->percent_aligned(val); + int code_offset = rec.code % PercentAligned; + get_read(metric, code_offset+1)->percent_aligned(val); } else INTEROP_THROW(bad_format_exception, "Unexpected tile code"); }; diff --git a/src/interop/model/run/info.cpp b/src/interop/model/run/info.cpp index 042116470..3c6133be1 100644 --- a/src/interop/model/run/info.cpp +++ b/src/interop/model/run/info.cpp @@ -20,6 +20,7 @@ #include "interop/util/xml_parser.h" #include "interop/io/metric_stream.h" #include "interop/logic/utils/enums.h" +#include "interop/logic/metric/tile_metric.h" using namespace illumina::interop::xml; @@ -44,7 +45,7 @@ namespace illumina { namespace interop { namespace model { namespace run } xml_node_ptr p_root = doc.first_node(); - if (p_root == 0) INTEROP_THROW(empty_xml_format_exception, "Root not found"); + if (p_root == 0) INTEROP_THROW(empty_xml_format_exception, "Root node not found"); if (p_root->name() != std::string("RunInfo")) INTEROP_THROW(bad_xml_format_exception, "Invalid run info xml file"); @@ -56,9 +57,14 @@ namespace illumina { namespace interop { namespace model { namespace run xml_node_ptr p_run_node = p_root->first_node(); - if (p_run_node == 0) INTEROP_THROW(bad_xml_format_exception, "Run not found"); + + for (; p_run_node; p_run_node = p_run_node->next_sibling()) + { + if(p_run_node->name() == std::string("Run"))break; + } + if (p_run_node == 0) INTEROP_THROW(bad_xml_format_exception, "Run node not found"); if (p_run_node->name() != std::string("Run")) - INTEROP_THROW(bad_xml_format_exception, "Invalid run info xml file"); + INTEROP_THROW(bad_xml_format_exception, "Invalid run info xml file: expected Run, got: " << p_run_node->name()); // Parse run attributes for (xml_attr_ptr attr = p_run_node->first_attribute(); @@ -138,7 +144,7 @@ namespace illumina { namespace interop { namespace model { namespace run } } } - + m_total_cycle_count = total_cycles(); } void info::read_file(const std::string &filename) throw(xml::xml_file_not_found_exception, @@ -149,8 +155,8 @@ namespace illumina { namespace interop { namespace model { namespace run { try { - rapidxml::file<> xmlFile(filename.c_str()); - parse(xmlFile.data()); + rapidxml::file<> xml_file(filename.c_str()); + parse(xml_file.data()); } catch (const std::runtime_error &ex) { INTEROP_THROW(xml_file_not_found_exception, ex.what()); @@ -170,6 +176,87 @@ namespace illumina { namespace interop { namespace model { namespace run } read_file(io::combine(run_folder, "RunInfo.xml")); } + /** Test if tile list matches flowcell layout + * + * @param lane lane number + * @param tile tile number + * @param read read number + * @throws invalid_run_info_exception + */ + void info::validate_read(const ::uint32_t lane, const ::uint32_t tile, const size_t read)const + throw(invalid_run_info_exception) + { + validate(lane, tile); + if(read > m_reads.size()) + INTEROP_THROW(invalid_run_info_exception, "Read number exceeds number of reads"); + } + /** Test if tile list matches flowcell layout + * + * @param lane lane number + * @param tile tile number + * @param cycle cycle number + * @throws invalid_run_info_exception + */ + void info::validate_cycle(const ::uint32_t lane, const ::uint32_t tile, const size_t cycle)const + throw(invalid_run_info_exception) + { + validate(lane, tile); + if(cycle > m_total_cycle_count) + INTEROP_THROW(invalid_run_info_exception, "Cycle number exceeds number of cycles: " + << cycle << " > " << m_total_cycle_count); + } + + /** Test if tile list matches flowcell layout + * + * @param lane lane number + * @param tile tile number + * @throws invalid_run_info_exception + */ + void info::validate(const ::uint32_t lane, const ::uint32_t tile)const throw(invalid_run_info_exception) + { + if(lane > m_flowcell.lane_count()) + INTEROP_THROW(invalid_run_info_exception,"Lane identifier exceeds number of lanes"); + const ::uint32_t swath = logic::metric::swath(tile, m_flowcell.naming_method()); + if(swath > m_flowcell.swath_count()) + INTEROP_THROW(invalid_run_info_exception, "Swath number exceeds number of swaths"); + const ::uint32_t tile_number = logic::metric::number(tile, m_flowcell.naming_method()); + if(tile_number > m_flowcell.tile_count()) + INTEROP_THROW(invalid_run_info_exception, "Tile number exceeds number of tiles"); + const ::uint32_t surface = logic::metric::surface(tile, m_flowcell.naming_method()); + if(surface > m_flowcell.surface_count()) + INTEROP_THROW(invalid_run_info_exception, "Surface number exceeds number of surfaces"); + const ::uint32_t section = logic::metric::section(tile, m_flowcell.naming_method()); + if(section > m_flowcell.sections_per_lane()) + INTEROP_THROW(invalid_run_info_exception, "Section number exceeds number of sections"); + } + + /** Test if tile list matches flowcell layout + * + * @throws invalid_run_info_exception + */ + void info::validate()const throw(invalid_run_info_exception, invalid_tile_naming_method) + { + typedef flowcell_layout::str_vector_t str_vector_t; + if(m_flowcell.naming_method()==constants::UnknownTileNamingMethod) + INTEROP_THROW(invalid_tile_naming_method, "Unknown tile naming method"); + std::set unique_numbers; + for(read_vector_t::const_iterator it = m_reads.begin();it != m_reads.end();++it) + { + if(unique_numbers.find(it->number()) != unique_numbers.end()) + INTEROP_THROW(invalid_run_info_exception, "Repeated read number"); + if (it->number() > m_reads.size()) + INTEROP_THROW(invalid_run_info_exception, "Missing reads"); + unique_numbers.insert(it->number()); + } + for(str_vector_t::const_iterator it = m_flowcell.tiles().begin();it != m_flowcell.tiles().end();++it) + { + const ::uint32_t lane = logic::metric::lane_from_name(*it); + if(lane == 0) INTEROP_THROW( invalid_run_info_exception, "Invalid tile identifier in tile names"); + const ::uint32_t tile = logic::metric::tile_from_name(*it); + if(tile == 0) INTEROP_THROW( invalid_run_info_exception, "Invalid tile identifier in tile names"); + validate(lane, tile); + } + } }}}} diff --git a/src/interop/model/run/parameters.cpp b/src/interop/model/run/parameters.cpp index 9842dad27..c748da576 100644 --- a/src/interop/model/run/parameters.cpp +++ b/src/interop/model/run/parameters.cpp @@ -24,125 +24,116 @@ #include "interop/logic/utils/enums.h" - using namespace illumina::interop::xml; -namespace illumina -{ -namespace interop -{ -namespace model -{ -namespace run -{ - -void parameters::parse(char *data) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception) +namespace illumina { namespace interop { namespace model { namespace run { - rapidxml::xml_document<> doc; - try{ - doc.parse<0>(data); - } - catch(const rapidxml::parse_error& ex) - { - INTEROP_THROW( xml_parse_exception, ex.what()); - } - m_version=0; - xml_node_ptr p_root = doc.first_node(); - if(p_root == 0) INTEROP_THROW( empty_xml_format_exception, "Root not found"); - if( p_root->name() != std::string("RunParameters")) - INTEROP_THROW( bad_xml_format_exception, "Invalid run parameters xml file"); - set_data_with_default(p_root->first_node("Version"), "Version", m_version, 0u); - - // Parse run data - std::string application_name; - std::string multi_surface; - m_instrument_type = constants::UnknownInstrument; - - xml_node_ptr p_setup = p_root->first_node("Setup"); - if(p_setup == 0) INTEROP_THROW( bad_xml_format_exception, "Setup not found"); - for(xml_node_ptr p_node = p_setup->first_node();p_node;p_node = p_node->next_sibling()) + void parameters::parse(char *data) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception) { - set_data(p_node, "ApplicationName", application_name); - set_data(p_node, "SupportMultipleSurfacesInUI", multi_surface); - } - set_instrument_id(application_name, multi_surface); + rapidxml::xml_document<> doc; + try + { + doc.parse<0>(data); + } + catch (const rapidxml::parse_error &ex) + { + INTEROP_THROW(xml_parse_exception, ex.what()); + } -} + m_version = 0; + xml_node_ptr p_root = doc.first_node(); + if (p_root == 0) INTEROP_THROW(empty_xml_format_exception, "Root not found"); + if (p_root->name() != std::string("RunParameters")) + INTEROP_THROW(bad_xml_format_exception, "Invalid run parameters xml file"); + set_data_with_default(p_root->first_node("Version"), "Version", m_version, 0u); + + // Parse run data + std::string application_name; + std::string multi_surface; + m_instrument_type = constants::UnknownInstrument; + + xml_node_ptr p_setup = p_root->first_node("Setup"); + if (p_setup == 0) INTEROP_THROW(bad_xml_format_exception, "Setup not found"); + for (xml_node_ptr p_node = p_setup->first_node(); p_node; p_node = p_node->next_sibling()) + { + set_data(p_node, "ApplicationName", application_name); + set_data(p_node, "SupportMultipleSurfacesInUI", multi_surface); + } + set_instrument_id(application_name, multi_surface); + } -void parameters::set_instrument_id(std::string& application_name, std::string& multi_surface) -{ - std::transform(application_name.begin(), application_name.end(), application_name.begin(), ::tolower); - std::transform(multi_surface.begin(), multi_surface.end(), multi_surface.begin(), ::tolower); - for(size_t i=0;i(i)); - std::transform(instrument_name.begin(), instrument_name.end(), instrument_name.begin(), ::tolower); - if(application_name.find(instrument_name) != std::string::npos) + std::transform(application_name.begin(), application_name.end(), application_name.begin(), ::tolower); + std::transform(multi_surface.begin(), multi_surface.end(), multi_surface.begin(), ::tolower); + + for (size_t i = 0; i < constants::InstrumentCount; ++i) { - m_instrument_type = static_cast(i); - break; + std::string instrument_name = constants::to_string(static_cast(i)); + std::transform(instrument_name.begin(), instrument_name.end(), instrument_name.begin(), ::tolower); + if (application_name.find(instrument_name) != std::string::npos) + { + m_instrument_type = static_cast(i); + break; + } + } + if (multi_surface != "" && m_instrument_type == constants::HiSeq) + { + // TODO: check if this even works! Find HiScan Example + if (multi_surface == "0" || multi_surface == "false" || multi_surface == "f") + m_instrument_type = constants::HiScan; } } - if(multi_surface != "" && m_instrument_type == constants::HiSeq) - { - // TODO: check if this even works! Find HiScan Example - if(multi_surface == "0" || multi_surface == "false" || multi_surface == "f") - m_instrument_type = constants::HiScan; - } -} -void parameters::read_file(const std::string &filename) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception) -{ - try - { - rapidxml::file<> xmlFile(filename.c_str()); - parse(xmlFile.data()); - }catch(const std::runtime_error& ex) + void parameters::read_file(const std::string &filename) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception) { - INTEROP_THROW( xml_file_not_found_exception, ex.what()); + try + { + rapidxml::file<> xml_file(filename.c_str()); + parse(xml_file.data()); + } + catch (const std::runtime_error &ex) + { + INTEROP_THROW(xml_file_not_found_exception, ex.what()); + } } -} -void parameters::read(const std::string& run_folder) throw( xml::xml_file_not_found_exception, - xml::bad_xml_format_exception, - xml::empty_xml_format_exception, - xml::missing_xml_element_exception, - xml::xml_parse_exception) -{ - - if(run_folder.find("RunParameters.xml") != std::string::npos || - run_folder.find("runParameters.xml") != std::string::npos) - { - read_file(run_folder); - return; - } - try + void parameters::read(const std::string &run_folder) throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception) { - read_file(io::combine(run_folder, "runParameters.xml")); - } - catch(const io::file_not_found_exception&) - { - read_file(io::combine(run_folder, "RunParameters.xml")); - } -} + if (run_folder.find("RunParameters.xml") != std::string::npos || + run_folder.find("runParameters.xml") != std::string::npos) + { + read_file(run_folder); + return; + } + try + { + read_file(io::combine(run_folder, "runParameters.xml")); + } + catch (const io::file_not_found_exception &) + { + read_file(io::combine(run_folder, "RunParameters.xml")); + } + } -} -} -} -} +}}}} #ifdef _MSC_VER diff --git a/src/interop/model/run_metrics.cpp b/src/interop/model/run_metrics.cpp new file mode 100644 index 000000000..80933b18a --- /dev/null +++ b/src/interop/model/run_metrics.cpp @@ -0,0 +1,544 @@ +/** Collection of all metric sets for a run + * + * @file + * @date 3/9/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include "interop/model/run_metrics.h" +#include "interop/io/metric_file_stream.h" + +#include "interop/logic/metric/q_metric.h" +#include "interop/logic/metric/tile_metric.h" +#include "interop/logic/utils/channel.h" + +namespace illumina { namespace interop { namespace model { namespace metrics +{ + struct populate_tile_list + { + populate_tile_list(run_metrics::tile_metric_map_t &map) : m_map(map) + {} + + template + void operator()(const MetricSet &metrics) const + { + typedef typename MetricSet::base_t base_t; + populate_id(metrics, base_t::null()); + } + + private: + template + void populate_id(const MetricSet &metrics, const void *) const + { + for (typename MetricSet::const_iterator it = metrics.begin(); it != metrics.end(); ++it) + { + INTEROP_ASSERTMSG(it->tile() > 0, it->lane() << "_" << it->tile()); + m_map[it->tile_hash()] = *it; + } + } + + template + void populate_id(const MetricSet &, const constants::base_lane_t *) const + {} + + run_metrics::tile_metric_map_t &m_map; + }; + + + struct populate_tile_cycle_list + { + populate_tile_cycle_list(run_metrics::cycle_metric_map_t &map) : m_map(map) + {} + + template + void operator()(const MetricSet &metrics) const + { + typedef typename MetricSet::base_t base_t; + populate_id(metrics, base_t::null()); + } + + private: + template + void populate_id(const MetricSet &metrics, const constants::base_cycle_t *) const + { + for (typename MetricSet::const_iterator it = metrics.begin(); it != metrics.end(); ++it) + { + INTEROP_ASSERTMSG(it->tile() > 0, it->lane() << "_" << it->tile() << " @ " << it->cycle()); + m_map[it->id()] = *it; + } + } + + template + void populate_id(const MetricSet &, const void *) const + {} + + run_metrics::cycle_metric_map_t &m_map; + }; + + struct is_metric_empty + { + is_metric_empty() : m_empty(true) + {} + + template + void operator()(const MetricSet &metrics) + { + if (metrics.size() > 0) m_empty = false; + } + + bool empty() const + { + return m_empty; + } + + bool m_empty; + }; + + struct check_for_each_data_source + { + check_for_each_data_source(const std::string &f) : m_run_folder(f) + {} + template + void operator()(MetricSet &metrics)const + { + metrics.data_source_exists(io::interop_exists(m_run_folder, metrics)); + } + std::string m_run_folder; + }; + struct clear_metric + { + template + void operator()(MetricSet &metrics)const + { + metrics.clear(); + } + }; + + struct read_func + { + read_func(const std::string &f) : m_run_folder(f) + {} + + template + int operator()(MetricSet &metrics) const + { + try + { + io::read_interop(m_run_folder, metrics); + } + catch (const io::file_not_found_exception &) + { + try + { + io::read_interop(m_run_folder, metrics, + false /** Search for XMetrics.bin not XMetricsOut.bin */); + } + catch (const io::file_not_found_exception &) + { return 1; } + catch (const io::incomplete_file_exception &) + { return 2; } + } + catch (const io::incomplete_file_exception &) + { return 2; } + return 0; + } + + std::string m_run_folder; + }; + + struct write_func + { + write_func(const std::string &f) : m_run_folder(f) + {} + + template + void operator()(const MetricSet &metrics) const + { + io::write_interop(m_run_folder, metrics); + } + + std::string m_run_folder; + }; + + struct check_if_groupid_is_empty + { + check_if_groupid_is_empty(const constants::metric_group group): m_empty(true), m_group(group) + {} + + template + void operator()(const MetricSet &metrics) + { + if(m_group == static_cast(MetricSet::TYPE)) + { + m_empty = metrics.empty(); + } + } + + bool empty() const + { + return m_empty; + } + + bool m_empty; + constants::metric_group m_group; + }; + struct check_if_group_is_empty + { + check_if_group_is_empty(const std::string &name) : m_empty(true), m_prefix(name) + {} + + template + void operator()(const MetricSet &metrics) + { + if(m_prefix == metrics.prefix()) + { + m_empty = metrics.empty(); + } + } + + bool empty() const + { + return m_empty; + } + + bool m_empty; + std::string m_prefix; + }; + + + + struct sort_by_lane_tile_cycle + { + template + void operator()(MetricSet &metrics) const + { + typedef typename MetricSet::metric_type metric_type; + std::sort(metrics.begin(), metrics.end(), is_less); + } + template + static bool is_less(const T& lhs, const T& rhs) + { + return lhs.id() < rhs.id(); + } + }; + + + struct validate_run_info + { + validate_run_info(const run::info& info) : m_info(info){} + + template + void operator()(const MetricSet &metrics)const + { + typedef typename MetricSet::base_t base_t; + validate(metrics, base_t::null()); + } + private: + template + void validate(const MetricSet &metrics, const constants::base_tile_t*)const + { + for(typename MetricSet::const_iterator it = metrics.begin();it != metrics.end();++it) + { + m_info.validate(it->lane(), it->tile()); + } + } + template + void validate(const MetricSet &metrics, const constants::base_cycle_t*)const + { + for(typename MetricSet::const_iterator it = metrics.begin();it != metrics.end();++it) + { + m_info.validate_cycle(it->lane(), it->tile(), it->cycle()); + } + } + template + void validate(const MetricSet &metrics, const constants::base_read_t*)const + { + for(typename MetricSet::const_iterator it = metrics.begin();it != metrics.end();++it) + { + m_info.validate_read(it->lane(), it->tile(), it->read()); + } + } + template + void validate(const MetricSet &, const void*)const{} + + const run::info& m_info; + }; + + /** Read binary metrics and XML files from the run folder + * + * @param run_folder run folder path + */ + void run_metrics::read(const std::string &run_folder) + throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception, + io::file_not_found_exception, + io::bad_format_exception, + io::incomplete_file_exception, + io::format_exception, + model::index_out_of_bounds_exception, + model::invalid_tile_naming_method, + model::invalid_run_info_exception) + { + read_metrics(run_folder); + const size_t count = read_xml(run_folder); + finalize_after_load(count); + } + + /** Read XML files: RunInfo.xml and possibly RunParameters.xml + * + * @param run_folder run folder path + */ + size_t run_metrics::read_xml(const std::string &run_folder) + throw(io::file_not_found_exception, + xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception) + { + read_run_info(run_folder); + return read_run_parameters(run_folder); + } + + /** Read RunInfo.xml + * + * @param run_folder run folder path + */ + void run_metrics::read_run_info(const std::string &run_folder) + throw(xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception) + { + m_run_info.read(run_folder); + } + + /** Read RunParameters.xml if necessary + * + * @param run_folder run folder path + */ + size_t run_metrics::read_run_parameters(const std::string &run_folder) throw(io::file_not_found_exception, + xml::xml_file_not_found_exception, + xml::bad_xml_format_exception, + xml::empty_xml_format_exception, + xml::missing_xml_element_exception, + xml::xml_parse_exception) + { + const size_t count = logic::metric::count_legacy_q_score_bins(get_set()); + if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count)) + { + + try + { + m_run_parameters.read(run_folder); + } + catch (const xml::xml_file_not_found_exception &) + { + if (m_run_info.channels().empty()) + INTEROP_THROW(io::file_not_found_exception, + "RunParameters.xml required for legacy run folders with missing channel names"); + else + INTEROP_THROW(io::file_not_found_exception, + "RunParameters.xml required for legacy run folders and is missing"); + } + } + return count; + } + + /** Finalize the metric sets after loading from disk + * + * @param count number of bins for legacy q-metrics + */ + void run_metrics::finalize_after_load(size_t count) + throw(io::format_exception, + model::invalid_tile_naming_method, + model::index_out_of_bounds_exception, model::invalid_run_info_exception) + { + if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) + { + m_run_info.set_naming_method( + logic::metric::tile_naming_method_from_metric(get_set())); + if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) + m_run_info.set_naming_method( + logic::metric::tile_naming_method_from_metric(get_set())); + if (m_run_info.flowcell().naming_method() == constants::UnknownTileNamingMethod) + m_run_info.set_naming_method( + logic::metric::tile_naming_method_from_metric(get_set())); + } + if (count == std::numeric_limits::max()) + { + if (get_set().size() > 0) + count = logic::metric::count_legacy_q_score_bins(get_set()); + else if (get_set().size()) + count = logic::metric::count_legacy_q_score_bins(get_set()); + } + logic::metric::populate_legacy_q_score_bins(get_set().bins(), m_run_parameters.instrument_type(), + count); + if (get_set().size() > 0 && get_set().size() == 0) + logic::metric::create_collapse_q_metrics(get_set(), get_set()); + if (get_set().size() > 0 && get_set().size() == 0) + logic::metric::create_q_metrics_by_lane(get_set(), get_set()); + logic::metric::populate_cumulative_distribution(get_set()); + logic::metric::populate_cumulative_distribution(get_set()); + logic::metric::populate_cumulative_distribution(get_set()); + INTEROP_ASSERT( + get_set().size() == 0 || get_set().size() == get_set().size()); + if (m_run_info.channels().empty()) + { + legacy_channel_update(m_run_parameters.instrument_type()); + if (m_run_info.channels().empty()) + INTEROP_THROW(io::format_exception, + "Channel names are missing from the RunInfo.xml, and RunParameters.xml does not contain sufficient information on the instrument run."); + } + if (!empty()) + { + if (run_info().flowcell().naming_method() == constants::UnknownTileNamingMethod) + INTEROP_THROW(model::invalid_tile_naming_method, + "Unknown tile naming method - update your RunInfo.xml"); + m_run_info.validate(); + validate(); + } + extraction_metric_set_t& extraction_metrics = get_set(); + // Trim excess channel data for imaging table + for (extraction_metric_set_t::iterator it = extraction_metrics.begin(); it != extraction_metrics.end(); ++it) + it->trim(run_info().channels().size()); + } + + /** Test if all metrics are empty + * + * @return true if all metrics are empty + */ + bool run_metrics::empty() const + { + is_metric_empty func; + m_metrics.apply(func); + return func.empty(); + } + /** Clear all the metrics + */ + void run_metrics::clear() + { + m_run_info = run::info(); + m_run_parameters = run::parameters(); + m_metrics.apply(clear_metric()); + } + + /** Update channels for legacy runs + * + * @param type instrument type + */ + void run_metrics::legacy_channel_update(const constants::instrument_type type) + { + m_run_info.channels(logic::utils::update_channel_from_instrument_type(type)); + } + + /** Set the tile naming method + * + * @param naming_method tile naming method + */ + void run_metrics::set_naming_method(const constants::tile_naming_method naming_method) + { + m_run_info.set_naming_method(naming_method); + } + + /** Read binary metrics from the run folder + * + * This function ignores: + * - Missing InterOp files + * - Incomplete InterOp files + * - Missing RunParameters.xml for non-legacy run folders + * + * @param run_folder run folder path + */ + void run_metrics::read_metrics(const std::string &run_folder) throw( + io::file_not_found_exception, + io::bad_format_exception, + io::incomplete_file_exception) + { + m_metrics.apply(read_func(run_folder)); + } + + /** Write binary metrics to the run folder + * + * @param run_folder run folder path + */ + void run_metrics::write_metrics(const std::string &run_folder) const + throw(io::file_not_found_exception, + io::bad_format_exception) + { + m_metrics.apply(write_func(run_folder)); + } + + /** Populate a map of valid tiles + * + * @param map mapping between tile has and base_metric + */ + void run_metrics::populate_id_map(tile_metric_map_t &map) const + { + m_metrics.apply(populate_tile_list(map)); + } + + /** Check if the metric group is empty + * + * @param group_name prefix of interop group metric + * @return true if metric is empty + */ + bool run_metrics::is_group_empty(const std::string& group_name) const + { + check_if_group_is_empty func(group_name); + m_metrics.apply(func); + return func.empty(); + } + /** Check if the metric group is empty + * + * @param group_id prefix of interop group metric id + * @return true if metric is empty + */ + bool run_metrics::is_group_empty(const constants::metric_group group_id) const + { + check_if_groupid_is_empty func(group_id); + m_metrics.apply(func); + return func.empty(); + } + + /** Populate a map of valid tiles and cycles + * + * @param map mapping between tile has and base_metric + */ + void run_metrics::populate_id_map(cycle_metric_map_t &map) const + { + m_metrics.apply(populate_tile_cycle_list(map)); + } + /** Validate whether the RunInfo.xml matches the InterOp files + * + * @throws invalid_run_info_exception + */ + void run_metrics::validate() throw(invalid_run_info_exception) + { + m_metrics.apply(validate_run_info(m_run_info)); + } + + /** Sort the metrics by lane, then tile, then cycle + * + */ + void run_metrics::sort() + { + m_metrics.apply(sort_by_lane_tile_cycle()); + } + + /** Check if the InterOp file for each metric set exists + * + * This will set the `metric_set::data_source_exists` flag. + * @param run_folder run folder path + */ + void run_metrics::check_for_data_sources(const std::string &run_folder) + { + m_metrics.apply(check_for_each_data_source(run_folder)); + } + + +}}}} diff --git a/src/interop/util/filesystem.cpp b/src/interop/util/filesystem.cpp new file mode 100644 index 000000000..a9be577ae --- /dev/null +++ b/src/interop/util/filesystem.cpp @@ -0,0 +1,133 @@ +/** Filesystem utility functions + * + * This header provides facilities to manipulate files, directories and the paths that identify them. + * + * @file + * @date 8/9/15 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include "interop/util/filesystem.h" + +#ifdef WIN32 +#include +/** Platform dependent path separator */ +#define INTEROP_OS_SEP '\\' +#else +#include +/** Platform dependent path separator */ +#define INTEROP_OS_SEP '/' +#endif + +#include +#include + +namespace illumina { namespace interop { namespace io +{ + /** Combine two directories or a directory and a filename into a file path + * + * This function provides a platform independent way to generate a file path. It currently supports most + * operating systems include Mac OSX, Windows and Linux/Unix. + * + * @param path string representing a file path + * @param name string representing a file or directory name to append to the end of the path + * @return proper os-dependent file path + */ + std::string combine(const std::string& path, const std::string& name) { + if (path != "" && path[path.size() - 1] != INTEROP_OS_SEP && name != "" && name[0] != INTEROP_OS_SEP) { + return path + INTEROP_OS_SEP + name; + } + return path + name; + } + namespace detail { +#ifndef WIN32 + /** Helper functor to match path separator on Linux + */ + struct match_path_sep + { + /** Test if given character is Linux path separator + * + * @param ch character to test + */ + bool operator()(char ch)const + { + return ch == '/'; + } + }; +#else + /** Helper functor to match path separator on Windows + */ + struct match_path_sep + { + /** Test if given character is Windows path separator + * + * @param ch character to test + */ + bool operator()(char ch)const + { + return ch == '/' || ch == '\\'; + } + }; +#endif + } + /** Get the file name from a file path + * + * @param source full file path + * @return name of the file + */ + std::string basename(std::string const& source) + { + if(source.empty()) return ""; + if( detail::match_path_sep()(source[source.length()-1] ) ) + { + return std::string(std::find_if(source.rbegin()+1, source.rend(), detail::match_path_sep()).base(), source.end()-1); + } + return std::string(std::find_if(source.rbegin(), source.rend(), detail::match_path_sep()).base(), source.end()); + } + /** Get the directory name from a file path + * + * @param source full file path + * @return name of the directory + */ + std::string dirname(std::string source) + { + if (source.size() <= 1) //Make sure it's possible to check the last character. + { + return source; + } + detail::match_path_sep is_sep; + if (is_sep(*(source.rbegin() + 1))) //Remove trailing slash if it exists. + { + source = source.substr(0, source.size()-1); + // source.pop_back(); // C++11 + } + source.erase(std::find_if(source.rbegin(), source.rend(), is_sep).base(), source.end()); + return source; + } + /** Check if a file exists + * + * @param filename name of the file + * @return true if the file exists and is readable + */ + bool is_file_readable(const std::string& filename) + { + std::ifstream fin(filename.c_str()); + return fin.good(); + } + /** Create a directory + * + * @param path path to new directory + * @return true if directory was created + */ + bool mkdir(const std::string& path, const int mode) + { +# ifdef WIN32 + (void)mode; + return _mkdir( path.c_str() ) == 0; +# else + return ::mkdir( path.c_str(), static_cast(mode)) == 0; +# endif + } +}}} + diff --git a/src/interop/util/time.cpp b/src/interop/util/time.cpp new file mode 100644 index 000000000..31f892c3d --- /dev/null +++ b/src/interop/util/time.cpp @@ -0,0 +1,163 @@ + +/** C# Time Conversion + * + * @file + * @date 1/14/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include "interop/util/time.h" + +#include +#include +#include "interop/util/static_assert.h" + +#ifdef HAVE_LONG64 +#define INTEROP_UTIL_TICKS_MASK 0x3fffffffffffffffL +#define INTEROP_UTIL_TICKS_THRESHOLD 0x3fffff36d5964000L +#define INTEROP_UTIL_TICKS_OFFSET 0x4000000000000000L +#define INTEROP_UTIL_TICKS_NEG_OFFSET 0xc92a69c000L +#define INTEROP_UTIL_TICKS_ENCODE 9223372036854775808ul +#define INTEROP_UTIL_TICKS_1970 621355968000000000ul +#else +#define INTEROP_UTIL_TICKS_MASK 0x3fffffffffffffffLL +#define INTEROP_UTIL_TICKS_THRESHOLD 0x3fffff36d5964000LL +#define INTEROP_UTIL_TICKS_OFFSET 0x4000000000000000LL +#define INTEROP_UTIL_TICKS_NEG_OFFSET 0xc92a69c000LL +#define INTEROP_UTIL_TICKS_ENCODE 9223372036854775808ull +#define INTEROP_UTIL_TICKS_1970 621355968000000000ull +#endif + +namespace illumina { namespace interop { namespace util +{ + + /** Constructor */ + csharp_date_time::csharp_date_time(const ::uint64_t v ) : value(v) { } + /** Convert to time_t unix format + * + * @return time_t unix format + */ + ::uint64_t csharp_date_time::to_unix()const + { + return to_unix(value); + } + /** Convert to to seconds with fractions + * + * @return seconds with fractions + */ + double csharp_date_time::to_seconds()const + { + return to_seconds(value); + } + + /** Convert to time_t unix format + * + * http://stackoverflow.com/questions/4014652/how-do-datetime-tobinary-and-datetime-tofiletime-differ + * + * @param val C# DateTime.ToBinary format + * @return time_t unix format + */ + ::uint64_t csharp_date_time::to_unix(const ::uint64_t val) + { + static_assert(sizeof(uint64_t) == 8, "Int64 has the wrong size"); + int64_t ticks = static_cast(val) & INTEROP_UTIL_TICKS_MASK; + if (ticks > INTEROP_UTIL_TICKS_THRESHOLD) + ticks -= INTEROP_UTIL_TICKS_OFFSET; + // TODO: missing conversion to local time (use encoded kind) + if (ticks < 0L) + { + ticks += INTEROP_UTIL_TICKS_NEG_OFFSET; + } + return static_cast( (ticks - ticks_to_1970()) / ticks_per_second() ); + } + /** Convert to seconds with fractions + * + * http://stackoverflow.com/questions/4014652/how-do-datetime-tobinary-and-datetime-tofiletime-differ + * + * @param val C# DateTime.ToBinary format + * @return seconds with fractions + */ + double csharp_date_time::to_seconds(const ::uint64_t val) + { + int64_t ticks = static_cast(val) & INTEROP_UTIL_TICKS_MASK; + if (ticks > INTEROP_UTIL_TICKS_THRESHOLD) + ticks -= INTEROP_UTIL_TICKS_OFFSET; + // TODO: missing conversion to local time (use encoded kind) + if (ticks < 0L) + { + ticks += INTEROP_UTIL_TICKS_NEG_OFFSET; + } + return (ticks - static_cast(ticks_to_1970())) / static_cast(ticks_per_second()); + } + /** Convert to c# format + * + * @param uval time_t unix format + * @return C# DateTime.ToBinary format + */ + csharp_date_time csharp_date_time::to_csharp(const ::uint64_t uval) + { + int64_t val = static_cast(uval); + val *= ticks_per_second(); + val += ticks_to_1970(); + if(val < 0l) val += INTEROP_UTIL_TICKS_OFFSET; + // TODO: missing conversion to local time (use encoded kind) + return csharp_date_time(static_cast(val | INTEROP_UTIL_TICKS_ENCODE));//-9223372036854775808 + } + /** Implicit convertion to an unsigned 64-bit integer + * + * @todo Wrap operator uint64_t in SWIG + * + * @return integer representation + */ + ::uint64_t csharp_date_time::to_binary()const + { + return value; + } + /** Test if the object is equal to another of the same type + * + * @param other other object being compared + * @return true if they are equal + */ + bool csharp_date_time::operator==(const csharp_date_time& other)const + { + const int64_t val = static_cast(value-other.value); + return ((val > 0) ? val : -val) < 5e14; + } + /** Write date type as integer to an output stream + * + * @param out output stream + * @param date_time data time object + * @return output stream + */ + std::ostream& operator<<(std::ostream& out, const csharp_date_time& date_time ) + { + // TODO: write out nice string representation + //2015-02-25 20:04:00 + out << date_time.value; + return out; + } + /** Read a date type from the input stream + * + * @param in input stream + * @param date_time data time object + * @return input stream + */ + std::istream& operator>>(std::istream& in, csharp_date_time& date_time) + { + // TODO: read in nice string representation + uint64_t val; + in >> val; + date_time = csharp_date_time(val); + return in; + } + + ::uint64_t csharp_date_time::ticks_per_second() + { + return 10000000; + } + ::uint64_t csharp_date_time::ticks_to_1970() + { + return INTEROP_UTIL_TICKS_1970; + } +}}} diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index c4576858d..672e017bd 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -4,14 +4,31 @@ add_custom_target(check) add_custom_target(performance) set_target_properties(check performance PROPERTIES EXCLUDE_FROM_ALL 1 EXCLUDE_FROM_DEFAULT_BUILD 1) - - add_subdirectory("interop") add_custom_command(TARGET check POST_BUILD COMMAND $) add_dependencies(check interop_gtests) -if(ENABLE_SWIG) +if(NOT ENABLE_SWIG) + return() +endif() + +find_package(SWIG) +if(NOT SWIG_FOUND) + return() +endif() +include(${PROJECT_SOURCE_DIR}/cmake/CheckSWIGVersion.cmake) + +check_swig_version(swig_major_ver) +if(swig_major_ver LESS "3") + return() +endif() + +if(NOT ENABLE_SWIG) + return() +endif() + +if(ENABLE_CSHARP) add_subdirectory("csharp") if(CSHARP_TEST_ENABLED) add_custom_command(TARGET check POST_BUILD diff --git a/src/tests/csharp/CMakeLists.txt b/src/tests/csharp/CMakeLists.txt index f46200c1a..6d864f2b8 100644 --- a/src/tests/csharp/CMakeLists.txt +++ b/src/tests/csharp/CMakeLists.txt @@ -11,16 +11,6 @@ if(NOT CSHARP_FOUND) return() endif() -find_package(SWIG) -if(NOT SWIG_FOUND) - return() -endif() - -check_swig_version(swig_major_ver) -if(swig_major_ver LESS "3") - return() -endif() - find_package(NUnit) if(NOT NUNIT_FOUND) message(WARNING "NUnit not found, C# unit testing will be disabled") @@ -49,13 +39,12 @@ set(TEST_SRCS metrics/QMetricsTest.cs metrics/TileMetricsTest.cs ) -set(csharp_files ${SWIG_GEN_CSHARP_SOURCE_FILES}) +set(csharp_files) foreach(SRC ${TEST_SRCS}) - get_filename_component(CSHARP_TEST_SRCS ${SRC} ABSOLUTE) - list(APPEND csharp_files ${CSHARP_TEST_SRCS}) + list(APPEND csharp_files "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}") endforeach() -csharp_add_library(csharp_unittest ${NUNIT_LIBRARIES} ${csharp_files}) +csharp_add_library(csharp_unittest ${NUNIT_LIBRARIES} ${csharp_files} ${SWIG_CSHARP_LIBRARY}) add_dependencies(csharp_unittest csharp_interop ${NUNIT_TARGET}) add_custom_command(TARGET csharp_unittest POST_BUILD @@ -75,13 +64,12 @@ endif() set(PERF_TEST_SRCS metrics/PerformanceTest.cs ) -set(csharp_perf_files ${SWIG_GEN_CSHARP_SOURCE_FILES}) +set(csharp_perf_files) foreach(SRC ${PERF_TEST_SRCS}) - get_filename_component(CSHARP_TEST_SRCS ${SRC} ABSOLUTE) - list(APPEND csharp_perf_files ${CSHARP_TEST_SRCS}) + list(APPEND csharp_perf_files "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}") endforeach() -csharp_add_library(csharp_perftest ${NUNIT_LIBRARIES} ${csharp_perf_files}) +csharp_add_library(csharp_perftest ${NUNIT_LIBRARIES} ${csharp_perf_files} ${SWIG_CSHARP_LIBRARY}) add_dependencies(csharp_perftest csharp_interop ${NUNIT_TARGET}) add_custom_command(TARGET csharp_perftest POST_BUILD diff --git a/src/tests/csharp/logic/ExceptionTest.cs b/src/tests/csharp/logic/ExceptionTest.cs index 6da4fd41c..98a43141a 100644 --- a/src/tests/csharp/logic/ExceptionTest.cs +++ b/src/tests/csharp/logic/ExceptionTest.cs @@ -5,6 +5,7 @@ using Illumina.InterOp.Metrics; using Illumina.InterOp.Plot; using Illumina.InterOp.Imaging; +using Illumina.InterOp.Comm; namespace Illumina.InterOp.Interop.UnitTest { @@ -22,7 +23,7 @@ public class ExceptionTests public void TestFileNotFoundException() { base_corrected_intensity_metrics metrics = new base_corrected_intensity_metrics(); - c_csharp_metrics.read_interop("/NO/FILE/EXISTS", metrics); + c_csharp_comm.read_interop("/NO/FILE/EXISTS", metrics); } ///

/// Test XmlFileNotFoundException @@ -55,7 +56,7 @@ public void TestInvalidFilterOption() filter_options options = new filter_options(tile_naming_method.FourDigit); candle_stick_plot_data data = new candle_stick_plot_data(); options.cycle(1); - c_csharp_plot.plot_candle_stick_by_cycle(metrics, metric_type.Intensity, options, data); + c_csharp_plot.plot_by_cycle(metrics, metric_type.Intensity, options, data); } /// /// Test invalid_metric_type @@ -67,19 +68,19 @@ public void TestInvalidMetricName() run_metrics metrics = new run_metrics(); filter_options options = new filter_options(tile_naming_method.FourDigit); candle_stick_plot_data data = new candle_stick_plot_data(); - c_csharp_plot.plot_candle_stick_by_cycle(metrics, "NoMetric", options, data); + c_csharp_plot.plot_by_cycle(metrics, "NoMetric", options, data); } /// /// Test invalid_metric_type /// - [Test] + /*[Test] [ExpectedException("Illumina.InterOp.Run.invalid_column_type")] public void TestInvalidColumnType() { size_vector_2d offsets = new size_vector_2d(); column_header_vector headers = new column_header_vector(); - headers.Add(new column_header("NoColumn")); + headers.Add(new column_header("NoColumn", column_data_type.UnknownDataType)); c_csharp_imaging.populate_column_offsets(offsets, headers); - } + }*/ } } diff --git a/src/tests/csharp/logic/ImagingTableLogic.cs b/src/tests/csharp/logic/ImagingTableLogic.cs index 7180cf8ef..9cb1c5b76 100644 --- a/src/tests/csharp/logic/ImagingTableLogic.cs +++ b/src/tests/csharp/logic/ImagingTableLogic.cs @@ -4,6 +4,7 @@ using Illumina.InterOp.Imaging; using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; +using Illumina.InterOp.Comm; namespace Illumina.InterOp.Interop.UnitTest { @@ -14,10 +15,10 @@ namespace Illumina.InterOp.Interop.UnitTest public class ImagingTableLogic { /// - /// Test plotting intensity by cycle + /// Test building a float backed image table /// [Test] - public void IntensityTest() + public void PopulateFloatBackedTableTest() { int[] tmp = new int[]{ 2,38 @@ -28,7 +29,53 @@ public void IntensityTest() byte[] expected_binary_data = new byte[tmp.Length]; for(int i=0;i + /// Test building a simple image table + /// + [Test] + public void PopulateTableTest() + { + int[] tmp = new int[]{ + 2,38 + ,7,0,90,4,1,0,-12,-56,15,64,-98,35,12,64,0,0,0,0,0,0,0,0,46,1,17,1,0,0,0,0,96,-41,-104,36,122,-86,-46,-120 + ,7,0,-66,4,1,0,96,-43,14,64,-63,49,13,64,0,0,0,0,0,0,0,0,56,1,17,1,0,0,0,0,112,125,77,38,122,-86,-46,-120 + ,7,0,66,8,1,0,74,-68,6,64,-118,-7,8,64,0,0,0,0,0,0,0,0,93,1,46,1,0,0,0,0,-47,-104,2,40,122,-86,-46,-120 + }; + byte[] expected_binary_data = new byte[tmp.Length]; + for(int i=0;i /// Test plotting intensity by cycle /// @@ -28,7 +35,7 @@ public void IntensityTest() byte[] expected_binary_data = new byte[tmp.Length]; for(int i=0;i 1 && !options.is_specific_surface()); + uint tile_count = layout.tiles_per_lane(); + float[] data_buffer = new float[lane_count*swath_count*tile_count]; + uint[] tile_buffer = new uint[lane_count*swath_count*tile_count]; flowcell_data data = new flowcell_data(); - c_csharp_plot.plot_flowcell_map(run, metric_type.QScore, options, data); + c_csharp_plot.plot_flowcell_map(run, metric_type.QScore, options, data, data_buffer, tile_buffer); Assert.AreEqual(data.row_count(), 8); } /// @@ -66,7 +73,7 @@ public void TestBadMetricException() byte[] expected_binary_data = new byte[tmp.Length]; for(int i=0;i; namespace Illumina.InterOp.Interop.UnitTest @@ -43,7 +44,7 @@ protected void SetupBuffers(int[] tmp, short version, q_score_header header) expected_binary_data = new byte[tmp.Length]; for(int i=0;i + class abstract_generator + { + public: + /** Generate the expected and actual metric sets + * + * @param expected expected object + * @param actual actual object + */ + virtual bool generate(T& expected, T& actual)=0; + /** Destructor */ + virtual ~abstract_generator(){} + }; + + + /** Generic test fixture + * + * This fixture will generated an expected and and actual object of any type. The objects are populated by + * parameters passed into GTest parameter interface. + */ + template + struct generic_test_fixture : public ::testing::TestWithParam< abstract_generator* > + { + private: + typedef ::testing::TestWithParam< abstract_generator* > parent_type; + public: + typedef abstract_generator* generator_type; + /** Value type of the object to test */ + typedef T value_type; + + /** Constructor */ + generic_test_fixture()//const char* test_dir) + { + INTEROP_ASSERT(0 != parent_type::GetParam()); + test = parent_type::GetParam()->generate(expected, actual); + } + /** Expected object to test */ + T expected; + /** Actual object to test */ + T actual; + /** Run test */ + bool test; + }; + + +}}} diff --git a/src/tests/interop/inc/proxy_parameter_generator.h b/src/tests/interop/inc/proxy_parameter_generator.h new file mode 100644 index 000000000..e499d430a --- /dev/null +++ b/src/tests/interop/inc/proxy_parameter_generator.h @@ -0,0 +1,155 @@ +/** Lazy parameter generation of a persistent std::vector + * + * @note This code may only work with gtest 1.7.0 + * + * @file + * @date 6/23/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include + + +namespace illumina{ namespace interop { namespace unittest { + /** Data structure that holds the parameters + * + * This holds a reference to the input vector, so ensure it is not on the stack! + * + * @note This code may only work with gtest 1.7.0 + */ + template + class proxy_argument_generator : public ::testing::internal::ParamGeneratorInterface< typename T::parent_type > + { + public: + /** Constructor + * + * @param obj object that acts like a functor + * @param vec reference to persistent vector of values + */ + proxy_argument_generator(T& obj, const std::vector& vec) : m_vec(vec), m_object(obj){} + /** Iterator to start of parameter collection + * + * @return iterator to start of parameter collection + */ + ::testing::internal::ParamIteratorInterface< typename T::parent_type >* Begin() const; + /** Iterator to end of parameter collection + * + * @return iterator to end of parameter collection + */ + ::testing::internal::ParamIteratorInterface< typename T::parent_type >* End() const; + + private: + const std::vector& m_vec; + T& m_object; + }; + /** Iterator over persistent vector of arguments + * + * @note This code may only work with gtest 1.7.0 + */ + template + class proxy_argument_iterator : public ::testing::internal::ParamIteratorInterface< typename T::parent_type > + { + typedef typename std::vector::const_iterator const_iterator; + public: + /** Constructor + * + * @param obj object-like functor + * @param base generator base + * @param it iterator to std::vector + */ + proxy_argument_iterator(T& obj, const proxy_argument_generator& base, const_iterator it) : m_base(base), m_begin(it), m_current(it), m_object(obj), p_value(&m_object) + { + } + /** Destructor */ + virtual ~proxy_argument_iterator() {} + /** A pointer to the base generator instance. + * Used only for the purposes of iterator comparison + * to make sure that two iterators belong to the same generator. + * + * @return base generator + */ + virtual const ::testing::internal::ParamGeneratorInterface< typename T::parent_type >* BaseGenerator() const + { + return &m_base; + } + /** Advances iterator to point to the next element + * provided by the generator. The caller is responsible + * for not calling Advance() on an iterator equal to + * BaseGenerator()->End(). + * + * + */ + virtual void Advance() + { + ++m_current; + } + /** Clones the iterator object. Used for implementing copy semantics + * of ParamIterator. + * + * @return new instance to object + */ + virtual ::testing::internal::ParamIteratorInterface< typename T::parent_type >* Clone() const + { + return new proxy_argument_iterator(*this); + } + /** Dereferences the current iterator and provides (read-only) access + * to the pointed value. It is the caller's responsibility not to call + * Current() on an iterator equal to BaseGenerator()->End(). + * Used for implementing ParamGenerator::operator*(). + * + * @return pointer to current value + */ + virtual const typename T::parent_type* Current() const + { + m_object(*m_current); + return &p_value; + } + /** Determines whether the given iterator and other point to the same + * element in the sequence generated by the generator. + * Used for implementing ParamGenerator::operator==(). + * + * @param other other iterator + * @return true if the base and iterator match + */ + virtual bool Equals(const ::testing::internal::ParamIteratorInterface< typename T::parent_type >& other) const + { + return &m_base == other.BaseGenerator() && + m_current == dynamic_cast*>(&other)->m_current; + } + private: + proxy_argument_iterator(const proxy_argument_iterator& other) + : m_base(other.m_base), m_begin(other.m_begin), + m_current(other.m_current), m_object(other.m_object), p_value(&m_object) {} + private: + const proxy_argument_generator& m_base; + const const_iterator m_begin; + const_iterator m_current; + T& m_object; + typename T::parent_type p_value; + }; + + + template + ::testing::internal::ParamIteratorInterface< typename T::parent_type >* proxy_argument_generator::Begin() const + { + return new proxy_argument_iterator(m_object, *this, m_vec.begin()); + } + template + ::testing::internal::ParamIteratorInterface< typename T::parent_type >* proxy_argument_generator::End() const + { + return new proxy_argument_iterator(m_object, *this, m_vec.end()); + } + + /** Generate parameters from a persistent vector (cannot be on the stack!) + * + * @note This code may only work with gtest 1.7.0 + * @param object object that acts like a functor + * @param values list of values + * @return parameter generator + */ + template + ::testing::internal::ParamGenerator< typename T::parent_type > ProxyValuesIn(T& object, const std::vector& values) { + return ::testing::internal::ParamGenerator< typename T::parent_type >(new proxy_argument_generator(object, values)); + } +}}} diff --git a/src/tests/interop/inc/regression_fixture.h b/src/tests/interop/inc/regression_fixture.h index 30ee8101a..9a871ed1c 100644 --- a/src/tests/interop/inc/regression_fixture.h +++ b/src/tests/interop/inc/regression_fixture.h @@ -10,94 +10,14 @@ #include #include #include -#include "persistent_parameter_generator.h" +#include "src/tests/interop/inc/persistent_parameter_generator.h" +#include "src/tests/interop/inc/regression_test_data.h" #include "interop/util/filesystem.h" #include "interop/model/run_metrics.h" -namespace illumina{ namespace interop { namespace unittest { - - - /** Singleton containing global data for all tests - */ - class regression_test_data - { - private: - regression_test_data() : m_rebaseline(false) - { - } - regression_test_data(const regression_test_data&){} - - public: - /** Get instance of the singleton - * - * @return instance - */ - static regression_test_data& instance() - { - static regression_test_data regression_test_data_instance; - return regression_test_data_instance; - } - - public: - /** Set the baseline file path - * - * @param filename baseline file path - */ - void baseline(const std::string& filename) - { - m_baseline = filename; - } - /** Get the baseline file path - * - * @return baseline file path - */ - const std::string& baseline()const - { - return m_baseline; - } - /** Add a file to the file list - * - * @param filename file name - */ - void add_file(const std::string& filename) - { - m_files.push_back(filename); - } - /** Get a vector of files to process - * - * @return file vector - */ - const std::vector< std::string >& files()const - { - return m_files; - } - /** Set the rebaseline flag - * - * If true, create a new baseline - * - * @param rebase rebaseline flag - */ - void rebaseline(const bool rebase) - { - m_rebaseline = rebase; - } - /** Test if new baseline should be created - * - * @return rebaseline flag state - */ - bool rebaseline()const - { - return m_rebaseline; - } - - private: - std::string m_baseline; - std::vector< std::string > m_files; - bool m_rebaseline; - - }; - +namespace illumina{ namespace interop { namespace unittest +{ /** Convert an array to a vector * diff --git a/src/tests/interop/inc/regression_test_data.h b/src/tests/interop/inc/regression_test_data.h new file mode 100644 index 000000000..b46c17026 --- /dev/null +++ b/src/tests/interop/inc/regression_test_data.h @@ -0,0 +1,100 @@ +/** Regression test data singleton + * + * + * @file + * @date 6/23/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include + +namespace illumina{ namespace interop { namespace unittest +{ + /** Singleton containing global data for all tests + */ + class regression_test_data + { + private: + regression_test_data() : m_rebaseline(false) + { + } + + regression_test_data(const regression_test_data &) + {} + + public: + /** Get instance of the singleton + * + * @return instance + */ + static regression_test_data &instance() + { + static regression_test_data regression_test_data_instance; + return regression_test_data_instance; + } + + public: + /** Set the baseline file path + * + * @param filename baseline file path + */ + void baseline(const std::string &filename) + { + m_baseline = filename; + } + + /** Get the baseline file path + * + * @return baseline file path + */ + const std::string &baseline() const + { + return m_baseline; + } + + /** Add a file to the file list + * + * @param filename file name + */ + void add_file(const std::string &filename) + { + m_files.push_back(filename); + } + + /** Get a vector of files to process + * + * @return file vector + */ + const std::vector &files() const + { + return m_files; + } + + /** Set the rebaseline flag + * + * If true, create a new baseline + * + * @param rebase rebaseline flag + */ + void rebaseline(const bool rebase) + { + m_rebaseline = rebase; + } + + /** Test if new baseline should be created + * + * @return rebaseline flag state + */ + bool rebaseline() const + { + return m_rebaseline; + } + + private: + std::string m_baseline; + std::vector m_files; + bool m_rebaseline; + }; +}}} diff --git a/src/tests/interop/logic/enum_parsing_test.cpp b/src/tests/interop/logic/enum_parsing_test.cpp index f77cd5599..22d7d9066 100644 --- a/src/tests/interop/logic/enum_parsing_test.cpp +++ b/src/tests/interop/logic/enum_parsing_test.cpp @@ -18,7 +18,10 @@ TEST(enum_parsing_test, parse_metric_type) EXPECT_EQ(constants::parse("Intensity"), constants::Intensity); EXPECT_EQ(constants::parse("FWHM"), constants::FWHM); EXPECT_EQ(constants::parse("Gobble"), constants::UnknownMetricType); +} +TEST(enum_parsing_test, list_enums_intensity) +{ std::vector types; constants::list_enums(types); EXPECT_EQ(types[0], constants::Intensity); diff --git a/src/tests/interop/logic/image_table_logic_test.cpp b/src/tests/interop/logic/image_table_logic_test.cpp deleted file mode 100644 index b1d54efec..000000000 --- a/src/tests/interop/logic/image_table_logic_test.cpp +++ /dev/null @@ -1,318 +0,0 @@ -/** Unit tests for image table logic - * - * @file - * @date 5/12/16 - * @version 1.0 - * @copyright GNU Public License. - */ -#include -#include -#include -#include "interop/logic/table/populate_imaging_table.h" -#include "interop/io/table/csv_format.h" -#include "src/tests/interop/metrics/inc/error_metrics_test.h" -#include "src/tests/interop/inc/regression_fixture.h" -#include "interop/model/table/imaging_table.h" -#include "interop/util/length_of.h" - -#define INTEROP_EXPECT_EQ_UIntIdType(EXPECTED, ACTUAL, TOL, MSG) ASSERT_EQ(EXPECTED, ACTUAL) << MSG -#define INTEROP_EXPECT_EQ_FloatValueType(EXPECTED, ACTUAL, TOL, MSG) if(!std::isnan(EXPECTED) && !std::isnan(ACTUAL)) \ - EXPECT_NEAR(EXPECTED, ACTUAL, TOL) << MSG -#define INTEROP_EXPECT_EQ_DateTimeStructType(EXPECTED, ACTUAL, TOL, MSG) EXPECT_TRUE(true) -#define INTEROP_EXPECT_EQ_UShortChannelArray(EXPECTED, ACTUAL, TOL, MSG) EXPECT_THAT(EXPECTED, ::testing::ElementsAreArray(ACTUAL)) << MSG -#define INTEROP_EXPECT_EQ_UShortBaseArray(EXPECTED, ACTUAL, TOL, MSG) EXPECT_THAT(EXPECTED, ::testing::ElementsAreArray(ACTUAL)) << MSG -#define INTEROP_EXPECT_EQ_FloatChannelArray(EXPECTED, ACTUAL, TOL, MSG) \ - ASSERT_EQ(EXPECTED.size(), ACTUAL.size()) << to_string(EXPECTED) << " - " << MSG;\ - for(size_t k=0;k column_header_vector_t; - -template -std::string to_string(const std::vector& values) -{ - std::ostringstream out; - for(size_t i=0;i& names = logic::table::imaging_table_column_names(); - for(size_t i=0;i channels; - channels.push_back("Red"); - channels.push_back("Green"); - std::vector headers; - std::vector filled(model::table::ImagingColumnCount, false); - filled[model::table::LaneColumn] = true; - filled[model::table::DensityKPermm2Column] = true; - filled[model::table::P90Column] = true; - filled[model::table::CycleWithinReadColumn] = true; - logic::table::populate_column_headers(headers, channels, filled); - const column_header expected_headers[] = { - column_header("", "Lane"), - column_header("", "Cycle Within Read"), - column_header("", "Density(k/mm2)"), - column_header("P90", "Red"), - column_header("P90", "Green")}; - EXPECT_THAT(to_vector(expected_headers), ::testing::ElementsAreArray(headers)); -} - -TEST(imaging_table, test_row0) -{ - const float tol = 1e-6f; - model::metrics::run_metrics metrics; - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(2, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); - metrics.legacy_channel_update(constants::HiSeq); - - std::istringstream iss(unittest::error_v3::binary_data()); - io::read_metrics(iss, metrics.error_metric_set()); - - std::vector table; - std::vector< bool > filled_columns; - logic::table::populate_imaging_table(metrics, table, filled_columns); - ASSERT_EQ(table.size(), 3u); - ASSERT_EQ(filled_columns.size(), static_cast(model::table::ImagingColumnCount)); - EXPECT_TRUE(filled_columns[0]); - EXPECT_TRUE(filled_columns[1]); - EXPECT_TRUE(filled_columns[model::table::ErrorRateColumn]); - EXPECT_FALSE(filled_columns[model::table::PercentPhasingColumn]); - EXPECT_NEAR(table[0].ErrorRate, 0.450f, tol); -} - -TEST(imaging_table, write_read_headers) -{ - typedef model::table::column_header column_header; - std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); - std::vector headers; - logic::table::populate_column_headers(headers, channels); - - std::ostringstream out; - io::table::write_csv_line(out, headers); - std::istringstream in(out.str()); - - std::vector actual_headers; - io::table::read_csv_line(in, actual_headers); - EXPECT_THAT(headers, ::testing::ElementsAreArray(actual_headers)); - -} -TEST(imaging_table, populate_column_offsets) -{ - typedef model::table::column_header column_header; - std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); - std::vector headers; - logic::table::populate_column_headers(headers, channels); - - std::ostringstream out; - io::table::write_csv_line(out, headers); - std::istringstream in(out.str()); - - std::vector actual_headers; - io::table::read_csv_line(in, actual_headers); - logic::table::table_fill_vector_t offsets; - logic::table::populate_column_offsets(offsets, actual_headers); - EXPECT_EQ(offsets[model::table::P90Column].size(), 2u); -} - -TEST(imaging_table, copy_to_vector) -{ - typedef model::table::column_header column_header; - const float tol = 1e-6f; - model::metrics::run_metrics metrics; - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(2, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); - - std::istringstream iss(unittest::error_v3::binary_data()); - io::read_metrics(iss, metrics.error_metric_set()); - - - std::vector table; - std::vector< bool > filled_columns; - logic::table::populate_imaging_table(metrics, table, filled_columns); - - std::vector values; - table[0].copy_to_vector(values, filled_columns);; - std::vector headers; - logic::table::populate_column_headers(headers, metrics.run_info().channels(), filled_columns); - - const model::metrics::error_metric metric = unittest::error_v3::metrics()[0]; - const float expected_values[] ={(float)metric.lane(), (float)metric.tile(), (float)metric.cycle(), 1, 1, roundf(metric.error_rate()*100)/100, 1, 1, 14}; - ASSERT_EQ(headers.size(), values.size()); - ASSERT_EQ(util::length_of(expected_values), values.size()); - for(size_t i=0;i(1, 0)); - std::vector tmp; - tmp.push_back(1u);tmp.push_back(2u); - offsets[model::table::P90Column] = tmp; - actual.copy_from_vector(std::vector(expected_values, expected_values+util::length_of(expected_values)), offsets); - - EXPECT_EQ(static_cast(expected_values[0]), actual.Lane); - EXPECT_EQ(static_cast(expected_values[1]), actual.P90[0]); - EXPECT_EQ(static_cast(expected_values[2]), actual.P90[1]); -} - - -TEST(imaging_table, read_csv_line ) -{ - std::istringstream in("1,2,,4\n"); - std::vector values; - io::table::read_csv_line(in, values); - EXPECT_EQ(values[0], 1u); - EXPECT_EQ(values[1], 2u); - EXPECT_EQ(values[2], 0u); - EXPECT_EQ(values[3], 4u); -} - - -//--------------------------------------------------------------------------------------------------------------------- -// Regression test section -//--------------------------------------------------------------------------------------------------------------------- -/** Imaging regression test fixture. - * - * This provides the test group name, provides the tested type as imaging_table, ensures the output file is prefixed with - * image_table and finally properly sets up the expected imaging_table object. - * - */ -struct image_table_regression_test : public regression_test_fixture< image_table_regression_test, model::table::imaging_table> -{ - image_table_regression_test() : regression_test_fixture< image_table_regression_test, model::table::imaging_table>("image_table"){} - /** Populate the actual imaging table using the given run_metrics - * - * @param actual_metrics run_metrics read in from a run_folder - * @param actual imaging table constructed from the run_metrics - */ - static void populate_actual(model::metrics::run_metrics& actual_metrics, model::table::imaging_table& actual) - { - logic::table::populate_imaging_table(actual_metrics, actual); - logic::table::populate_column_headers(actual_metrics.run_info().channels(), actual); - } -}; - -//@todo remove this function -bool compare_rows(const model::table::table_entry& lhs, const model::table::table_entry& rhs) -{ - if(lhs.Lane == rhs.Lane) - { - if(lhs.Tile == rhs.Tile) return lhs.Cycle < rhs.Cycle; - return lhs.Tile < rhs.Tile; - } - return lhs.Lane < rhs.Lane; -} - -TEST_P(image_table_regression_test, compare_to_baseline) -{ - if(!test) return; - ASSERT_EQ(expected.column_count(), actual.column_count()) << "Baseline: " << to_string(expected.headers()) << "\n Actual: " << to_string(actual.headers()); - ASSERT_EQ(expected.row_count(), actual.row_count()); - EXPECT_THAT(expected.headers(), ::testing::ElementsAreArray(actual.headers())); - // @todo remove the following sort - std::stable_sort(expected.begin(), expected.end(), compare_rows); - std::stable_sort(actual.begin(), actual.end(), compare_rows); - - // Account for differences in rounding due to writing floating-point numbers as text (and double/float issues) - // C# rounds to nearest even - // InterOp rounds using std::floor(val+0.5) - const float round_tol = 0.101f; - const float round_tols[] = {1e-5f, round_tol, round_tol/10, round_tol/100}; - - for(size_t i=0;i +#include +#include "interop/util/length_of.h" +#include "interop/logic/table/create_imaging_table.h" +#include "src/tests/interop/metrics/inc/error_metrics_test.h" +#include "src/tests/interop/inc/regression_fixture.h" +#include "interop/io/table/imaging_table_csv.h" + +using namespace illumina::interop; +using namespace illumina::interop::unittest; + +namespace illumina{ namespace interop {namespace model {namespace table +{ + /** Compare whether two imaging columns are equal + * + * @param lhs an imaging column + * @param rhs an imaging column + * @return true if the columns are equal + */ + bool operator==(const imaging_column& lhs, const imaging_column& rhs) + { + if(lhs.subcolumns().size() != rhs.subcolumns().size()) return false; + for(size_t i=0;i reads(2); + reads[0] = model::run::read_info(1, 1, 26); + reads[1] = model::run::read_info(2, 27, 76); + metrics.run_info(model::run::info( + "", + "", + 1, + model::run::flowcell_layout(2, 2, 2, 16), + std::vector(), + model::run::image_dimensions(), + reads + )); + metrics.legacy_channel_update(constants::HiSeq); + metrics.set_naming_method(constants::FourDigit); + + std::istringstream iss(unittest::error_v3::binary_data()); + io::read_metrics(iss, metrics.get_set()); +} +/** + * @class illumina::interop::model::table::imaging_table + * @test Confirm to_header and to_name properly convert header names + */ +TEST(imaging_table, convert_name_to_header) +{ + for(size_t i=0;i(i)); + const std::string header = model::table::imaging_column::to_header(expected_name); + const std::string actual_name = model::table::imaging_column::to_name(header); + EXPECT_EQ(expected_name, actual_name); + } +} + +TEST(imaging_table, base_header_test) +{ + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + std::vector filled(model::table::ImagingColumnCount, false); + + filled[model::table::PercentBaseColumn] = true; + std::vector< model::table::imaging_column > columns; + logic::table::create_imaging_table_columns(channels, filled, columns); + + const std::string expected_subcolumns[] = { + "A", "C", "G", "T" + }; + EXPECT_THAT(to_vector(expected_subcolumns), ::testing::ElementsAreArray(columns[0].subcolumns())); +} + +TEST(imaging_table, test_write_column_name) +{ + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + std::vector filled(model::table::ImagingColumnCount, false); + + filled[model::table::PercentBaseColumn] = true; + std::vector< model::table::imaging_column > columns; + logic::table::create_imaging_table_columns(channels, filled, columns); + + std::ostringstream oss; + oss << columns[0]; + EXPECT_EQ(oss.str(), "% Base"); +} + +TEST(imaging_table, test_read_column_name) +{ + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + std::vector filled(model::table::ImagingColumnCount, false); + + filled[model::table::PercentBaseColumn] = true; + std::vector< model::table::imaging_column > columns; + logic::table::create_imaging_table_columns(channels, filled, columns); + + std::vector< model::table::imaging_column > actual_columns(1); + std::istringstream iss("% Base"); + iss >> actual_columns[0]; + logic::table::populate_column_offsets(actual_columns); + EXPECT_EQ(actual_columns[0], columns[0]); +} + +TEST(imaging_table, read_csv_line ) +{ + std::istringstream in("1,2,,4\n"); + std::vector values; + io::table::read_csv_line(in, values); + EXPECT_EQ(values[0], 1u); + EXPECT_EQ(values[1], 2u); + EXPECT_EQ(values[2], 0u); + EXPECT_EQ(values[3], 4u); +} + + +TEST(imaging_table, create_imaging_table_columns_channel) +{ + std::vector channels; + channels.push_back("Red"); + channels.push_back("Green"); + std::vector filled(model::table::ImagingColumnCount, false); + + filled[model::table::P90Column] = true; + std::vector< model::table::imaging_column > columns; + logic::table::create_imaging_table_columns(channels, filled, columns); + + std::vector< model::table::imaging_column > expected_columns; + expected_columns.push_back(model::table::imaging_column(model::table::P90Column, + 0, + channels)); + + EXPECT_THAT(expected_columns, ::testing::ElementsAreArray(columns)); +} + +TEST(imaging_table, create_imaging_table_columns_error_metrics) +{ + model::metrics::run_metrics metrics; + simulate_read_error_metrics(metrics); + + std::vector columns; + logic::table::create_imaging_table_columns(metrics, columns); + + const model::table::imaging_column expected_columns[] = { + model::table::imaging_column(model::table::LaneColumn, 0), + model::table::imaging_column(model::table::TileColumn, 1), + model::table::imaging_column(model::table::CycleColumn, 2), + model::table::imaging_column(model::table::ReadColumn, 3), + model::table::imaging_column(model::table::CycleWithinReadColumn, 4), + model::table::imaging_column(model::table::ErrorRateColumn, 5), + model::table::imaging_column(model::table::SurfaceColumn, 6), + model::table::imaging_column(model::table::SwathColumn, 7), + model::table::imaging_column(model::table::TileNumberColumn, 8) + }; + + EXPECT_THAT(to_vector(expected_columns), ::testing::ElementsAreArray(columns)); +} +TEST(imaging_table, create_imaging_table_error_metrics) +{ + model::metrics::run_metrics metrics; + simulate_read_error_metrics(metrics); + + std::vector columns; + std::map row_offsets; + logic::table::create_imaging_table_columns(metrics, columns); + const size_t column_count = logic::table::count_table_columns(columns); + logic::table::count_table_rows(metrics, row_offsets); + std::vector data(row_offsets.size()*column_count); + ASSERT_TRUE(data.size() > 0); + logic::table::populate_imaging_table_data(metrics, columns, row_offsets, &data[0], data.size()); + EXPECT_EQ(data[0], 7u); +} +TEST(imaging_table, create_imaging_table_error_metrics_tile_test) +{ + model::metrics::run_metrics metrics; + simulate_read_error_metrics(metrics); + + model::table::imaging_table table; + logic::table::create_imaging_table(metrics, table); + EXPECT_EQ(table(0, model::table::LaneColumn), 7.0f); + EXPECT_EQ(table(0, model::table::TileColumn), 1114.0f); + EXPECT_EQ(table(0, model::table::CycleColumn), 1.0f); + EXPECT_EQ(table(0, model::table::TileNumberColumn), 14.0f); + EXPECT_EQ(table(0, model::table::SurfaceColumn), 1.0f); + EXPECT_EQ(table(0, model::table::SwathColumn), 1.0f); + + + EXPECT_EQ(table(1, model::table::LaneColumn), 7.0f); + EXPECT_EQ(table(1, model::table::TileColumn), 1114.0f); + EXPECT_EQ(table(1, model::table::CycleColumn), 2.0f); + EXPECT_EQ(table(1, model::table::TileNumberColumn), 14.0f); + EXPECT_EQ(table(1, model::table::SurfaceColumn), 1.0f); + EXPECT_EQ(table(1, model::table::SwathColumn), 1.0f); +} + diff --git a/src/tests/interop/logic/imaging_table_regression_test.cpp b/src/tests/interop/logic/imaging_table_regression_test.cpp new file mode 100644 index 000000000..1e354a685 --- /dev/null +++ b/src/tests/interop/logic/imaging_table_regression_test.cpp @@ -0,0 +1,109 @@ +/** Regression tests for image table logic + * + * @file + * @date 7/21/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include +#include "interop/util/math.h" +#include "interop/util/length_of.h" +#include "interop/logic/table/create_imaging_table.h" +#include "src/tests/interop/metrics/inc/error_metrics_test.h" +#include "src/tests/interop/inc/regression_fixture.h" +#include "interop/io/table/imaging_table_csv.h" + +using namespace illumina::interop; +using namespace illumina::interop::unittest; + +namespace illumina{ namespace interop {namespace model {namespace table +{ + bool operator==(const imaging_column& lhs, const imaging_column& rhs); +}}}} + +template +std::string to_string(const std::vector& values) +{ + std::ostringstream out; + for(size_t i=0;i +{ + /** Constructor */ + imaging_table_regression_test() : regression_test_fixture< imaging_table_regression_test, model::table::imaging_table>("image_table"){} + /** Populate the actual imaging table using the given run_metrics + * + * @param actual_metrics run_metrics read in from a run_folder + * @param actual imaging table constructed from the run_metrics + */ + static void populate_actual(model::metrics::run_metrics& actual_metrics, model::table::imaging_table& actual) + { + logic::table::create_imaging_table(actual_metrics, actual); + } +}; + + +TEST_P(imaging_table_regression_test, compare_to_baseline) +{ + if(!test) return; + ASSERT_EQ(expected.column_count(), actual.column_count()) << "Baseline: " << to_string(expected.columns()) << "\n Actual: " << to_string(actual.columns()); + ASSERT_EQ(expected.row_count(), actual.row_count()); + EXPECT_THAT(expected.columns(), ::testing::ElementsAreArray(actual.columns())); + std::vector expected_idx(expected.row_count(), expected.row_count()); + for(size_t i=0;i actual_idx(expected_idx); + + + // Account for differences in rounding due to writing floating-point numbers as text (and double/float issues) + // C# rounds to nearest even + // InterOp rounds using std::floor(val+0.5) + const float round_tol = 0.101f; + const float round_tols[] = {1e-5f, round_tol, round_tol/10, round_tol/100}; + +// @todo remove the following sort + std::stable_sort(expected_idx.begin(), expected_idx.end(), model::table::imaging_table_id_less(expected)); + std::stable_sort(actual_idx.begin(), actual_idx.end(), model::table::imaging_table_id_less(actual)); + for(size_t row_index=0;row_index #include "interop/model/plot/filter_options.h" #include "interop/logic/plot/plot_by_cycle.h" @@ -16,6 +20,42 @@ using namespace illumina::interop; +TEST(plot_logic, check_plot_by_cycle_list) +{ + std::vector types; + logic::plot::list_by_cycle_metrics(types); + for(size_t i=0;i types; + logic::plot::list_by_lane_metrics(types); + for(size_t i=0;i types; + logic::plot::list_flowcell_metrics(types); + for(size_t i=0;i(), model::run::image_dimensions(), reads @@ -152,7 +192,7 @@ TEST(plot_logic, q_score_heatmap) "", "", 1, - model::run::flowcell_layout(2, 2, 2, 16), + model::run::flowcell_layout(8, 2, 2, 16), std::vector(), model::run::image_dimensions(), reads diff --git a/src/tests/interop/metrics/base_metric_tests.cpp b/src/tests/interop/metrics/base_metric_tests.cpp index 08829d056..0693d39b9 100644 --- a/src/tests/interop/metrics/base_metric_tests.cpp +++ b/src/tests/interop/metrics/base_metric_tests.cpp @@ -8,21 +8,77 @@ */ #include #include "interop/model/metric_base/base_metric.h" +#include "interop/model/metric_base/base_cycle_metric.h" +#include "interop/model/metric_base/base_read_metric.h" #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants #endif +using namespace illumina::interop::model; using namespace illumina::interop::model::metric_base; - TEST(base_metric_test, lane_from_id) { - base_metric::id_t id = base_metric::id(8, 1323); + base_metric::id_t id = base_metric::create_id(8, 1323); EXPECT_EQ(base_metric::lane_from_id(id), 8u); - - base_metric::id_t id2 = base_metric::id(1, 1); + base_metric::id_t id2 = base_metric::create_id(1, 1); EXPECT_EQ(base_metric::lane_from_id(id2), 1u); } +TEST(base_metric_test, tile_from_id) +{ + base_metric::id_t id = base_metric::create_id(8, 1323); + EXPECT_EQ(base_metric::tile_from_id(id), 1323u); +} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +TEST(base_cycle_metric_test, lane_from_id) +{ + base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 2); + EXPECT_EQ(base_cycle_metric::lane_from_id(id), 8u); + + base_metric::id_t id2 = base_cycle_metric::create_id(1, 1, 2); + EXPECT_EQ(base_cycle_metric::lane_from_id(id2), 1u); +} +TEST(base_cycle_metric_test, tile_from_id) +{ + base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 2); + EXPECT_EQ(base_cycle_metric::tile_from_id(id), 1323u); +} +TEST(base_cycle_metric_test, cycle_from_id) +{ + base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 10); + EXPECT_EQ(base_cycle_metric::cycle_from_id(id), 10u); +} +TEST(base_cycle_metric_test, tile_hash_from_id) +{ + base_metric::id_t id = base_cycle_metric::create_id(8, 1323, 10); + EXPECT_EQ(base_cycle_metric::tile_hash_from_id(id), base_metric::create_id(8, 1323)); +} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +TEST(base_read_metric_test, lane_from_id) +{ + base_metric::id_t id = base_read_metric::create_id(8, 1323, 2); + EXPECT_EQ(base_read_metric::lane_from_id(id), 8u); + + base_metric::id_t id2 = base_read_metric::create_id(1, 1, 2); + EXPECT_EQ(base_read_metric::lane_from_id(id2), 1u); +} +TEST(base_read_metric_test, tile_from_id) +{ + base_metric::id_t id = base_read_metric::create_id(8, 1323, 2); + EXPECT_EQ(base_read_metric::tile_from_id(id), 1323u); +} +TEST(base_read_metric_test, read_from_id) +{ + base_metric::id_t id = base_read_metric::create_id(8, 1323, 3); + EXPECT_EQ(base_read_metric::read_from_id(id), 3u); +} +TEST(base_read_metric_test, tile_hash_from_id) +{ + base_metric::id_t id = base_read_metric::create_id(8, 1323, 10); + EXPECT_EQ(base_read_metric::tile_hash_from_id(id), base_metric::create_id(8, 1323)); +} + + diff --git a/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp b/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp index de78ddf39..d846c189f 100644 --- a/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp +++ b/src/tests/interop/metrics/corrected_intensity_metrics_test.cpp @@ -15,6 +15,7 @@ #include #include "interop/util/math.h" #include "inc/corrected_intensity_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop; using namespace illumina::interop::io; @@ -37,34 +38,35 @@ TYPED_TEST_CASE(corrected_intensity_metrics_test, Formats); template void compare_metrics(const T& actual, const T& expected) { + const float tol = 1e-3f; EXPECT_EQ(actual.version(), expected.version()); EXPECT_EQ(actual.size(), expected.size()); EXPECT_EQ(actual.max_cycle(), expected.max_cycle()); - for(typename T::const_iterator itExpected=expected.begin(), itActual = actual.begin(); - itExpected != expected.end() && itActual != actual.end(); - itExpected++,itActual++) + for(typename T::const_iterator it_expected=expected.begin(), it_actual = actual.begin(); + it_expected != expected.end() && it_actual != actual.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); if(expected.version() < 3) { - EXPECT_EQ(itExpected->averageCycleIntensity(), itActual->averageCycleIntensity()); + EXPECT_EQ(it_expected->average_cycle_intensity(), it_actual->average_cycle_intensity()); } if(expected.version() == 2) { - if (!std::isnan(itExpected->signalToNoise()) || !std::isnan(itActual->signalToNoise())) - EXPECT_NEAR(itExpected->signalToNoise(), itActual->signalToNoise(), 1e-7f); + if (!std::isnan(it_expected->signal_to_noise()) || !std::isnan(it_actual->signal_to_noise())) + EXPECT_NEAR(it_expected->signal_to_noise(), it_actual->signal_to_noise(), tol); } for(ptrdiff_t i=-1;icalledCounts(i), itActual->calledCounts(i)); + EXPECT_EQ(it_expected->called_counts(static_cast(i)), it_actual->called_counts(static_cast(i))); for(size_t i=0;icorrectedIntCalled(i), itActual->correctedIntCalled(i)); + EXPECT_EQ(it_expected->corrected_int_called(static_cast(i)), it_actual->corrected_int_called(static_cast(i))); if(expected.version() < 3) { - EXPECT_EQ(itExpected->correctedIntAll(i), itActual->correctedIntAll(i)); + EXPECT_EQ(it_expected->corrected_int_all(static_cast(i)), it_actual->corrected_int_all(static_cast(i))); } } } @@ -90,6 +92,16 @@ TEST(corrected_intensity_metrics_test, test_percent_base_nan) EXPECT_TRUE(std::isnan(metric.percent_bases()[0])); } + +TEST(run_metrics_corrected_intensity_test, test_is_group_empty) +{ + run_metrics metrics; + std::istringstream fin(corrected_intensity_v2::binary_data()); + EXPECT_TRUE(metrics.is_group_empty(constants::CorrectedInt)); + io::read_metrics(fin, metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::CorrectedInt)); +} + #define FIXTURE corrected_intensity_metrics_test /** * @class illumina::interop::model::metrics::corrected_intensity_metric diff --git a/src/tests/interop/metrics/error_metrics_test.cpp b/src/tests/interop/metrics/error_metrics_test.cpp index 5d63780ee..89dcac60d 100644 --- a/src/tests/interop/metrics/error_metrics_test.cpp +++ b/src/tests/interop/metrics/error_metrics_test.cpp @@ -2,7 +2,6 @@ * * * @file - * * @date 8/23/2015 * @version 1.0 * @copyright GNU Public License. @@ -11,6 +10,7 @@ #include #include #include "inc/error_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; using namespace illumina::interop::io; @@ -31,28 +31,41 @@ TYPED_TEST_CASE(error_metrics_test, Formats); */ TYPED_TEST(error_metrics_test, test_read_write) { + const float tol = 1e-4f; EXPECT_EQ(TypeParam::actual_metric_set.version(), TypeParam::VERSION); EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); EXPECT_EQ(TypeParam::actual_metric_set.max_cycle(), TypeParam::expected_metric_set.max_cycle()); - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); - EXPECT_EQ(itExpected->mismatch_count(), itActual->mismatch_count()); - EXPECT_NEAR(itExpected->errorRate(), itActual->errorRate(), 1e-7f); - for(ptrdiff_t i=0;i(itExpected->mismatch_count());i++) - EXPECT_EQ(itExpected->mismatch_cluster_count(i), itActual->mismatch_cluster_count(i)); + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->mismatch_count(), it_actual->mismatch_count()); + EXPECT_NEAR(it_expected->error_rate(), it_actual->error_rate(), tol); + for(ptrdiff_t i=0;i(it_expected->mismatch_count());i++) + EXPECT_EQ(it_expected->mismatch_cluster_count(i), it_actual->mismatch_cluster_count(i)); } } TYPED_TEST(error_metrics_test, test_tile_metric_count_for_lane) { - EXPECT_EQ(TypeParam::expected_metric_set.tile_numbers_for_lane(7).size(), 1u); + ASSERT_GT(TypeParam::expected_metric_set.size(), 1u); + const error_metric::uint_t lane = TypeParam::expected_metric_set.metrics()[0].lane(); + EXPECT_EQ(TypeParam::expected_metric_set.tile_numbers_for_lane(lane).size(), 1u); +} + + +TEST(run_metrics_error_test, test_is_group_empty) +{ + run_metrics metrics; + std::istringstream fin(error_v3::binary_data()); + EXPECT_TRUE(metrics.is_group_empty(constants::Error)); + io::read_metrics(fin, metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::Error)); } #define FIXTURE error_metrics_test diff --git a/src/tests/interop/metrics/error_metrics_test2.cpp b/src/tests/interop/metrics/error_metrics_test2.cpp new file mode 100644 index 000000000..6fe3337c6 --- /dev/null +++ b/src/tests/interop/metrics/error_metrics_test2.cpp @@ -0,0 +1,127 @@ +/** Unit tests for the error metrics + * + * + * @file + * @date 8/23/2015 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include +#include +#include "src/tests/interop/inc/generic_fixture.h" +#include "src/tests/interop/inc/proxy_parameter_generator.h" +#include "src/tests/interop/metrics/inc/metric_generator.h" +#include "src/tests/interop/metrics/inc/error_metrics_test.h" +#include "interop/model/run_metrics.h" +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::metric_base; +using namespace illumina::interop::io; +using namespace illumina::interop; +using namespace illumina::interop::unittest; + +typedef metric_set< error_metric > error_metric_set; +/** Setup for tests that compare two error metric sets */ +struct error_metrics_tests : public generic_test_fixture< error_metric_set > {}; + +error_metrics_tests::generator_type unit_test_generators[] = { + new hardcoded_metric_generator< error_metric_v3 >() , + new write_read_metric_generator< error_metric_v3 >() +}; + +// Setup unit tests for error_metrics_tests +INSTANTIATE_TEST_CASE_P(error_metric_unit_test, + error_metrics_tests, + ::testing::ValuesIn(unit_test_generators)); + +/** + * @class illumina::interop::model::metrics::error_metric + * @test Confirm version 3 of the metric can be written to and read from a stream + * @test Confirm version 3 of the metric matches known binary file + */ +TEST_P(error_metrics_tests, compare_expected_actual) +{ + if(!test) return;// Disable test for rebaseline + EXPECT_EQ(actual.version(), expected.version()); + EXPECT_EQ(actual.size(), expected.size()); + EXPECT_EQ(actual.max_cycle(), expected.max_cycle()); + + for(value_type::const_iterator it_expected=expected.begin(), it_actual = actual.begin(); + it_expected != expected.end() && it_actual != actual.end(); + it_expected++,it_actual++) + { + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->mismatch_count(), it_actual->mismatch_count()); + EXPECT_NEAR(it_expected->error_rate(), it_actual->error_rate(), 1e-7f); + for(ptrdiff_t i=0;i(it_expected->mismatch_count());i++) + EXPECT_EQ(it_expected->mismatch_cluster_count(i), it_actual->mismatch_cluster_count(i)); + } +} + +/** + * @test Ensure tile_numbers_for_lane returns the proper number + */ +TEST(error_metrics_single_test, test_tile_metric_count_for_lane) +{ + error_metric_set metrics; + error_metric_v3::create_expected(metrics); + EXPECT_EQ(metrics.tile_numbers_for_lane(7).size(), 1u); +} + +/** + * @test Ensure the keys function returns the proper metric + */ +TEST(error_metrics_single_test, test_expected_get_metric) +{ + error_metric_set metrics; + error_metric_v3::create_expected(metrics); + error_metric_set::key_vector keys = metrics.keys(); + for(size_t i=0;i()); + EXPECT_FALSE(metrics.is_group_empty(constants::Error)); +} + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Setup regression test +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +regression_test_metric_generator regression_gen("metrics"); +INSTANTIATE_TEST_CASE_P(error_metric_regression_test, + error_metrics_tests, + ProxyValuesIn(regression_gen, regression_test_data::instance().files())); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Shared stream tests +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** Setup for tests that compare two binary buffers created by writing error metrics */ +struct error_metric_write_tests : public generic_test_fixture< std::string > +{ + /** Define a metric set type */ + typedef error_metric_set metric_set_t; +}; + +error_metric_write_tests::generator_type write_generators[] = { + new write_metric_generator< error_metric_v3 >() +}; +INSTANTIATE_TEST_CASE_P(error_metric_write_test_params, + error_metric_write_tests, + ::testing::ValuesIn(write_generators)); + + +#define FIXTURE error_metric_write_tests +#include "src/tests/interop/metrics/inc/metric_stream_tests.hpp" + diff --git a/src/tests/interop/metrics/extraction_metrics_test.cpp b/src/tests/interop/metrics/extraction_metrics_test.cpp index d48771b69..d44a8e535 100644 --- a/src/tests/interop/metrics/extraction_metrics_test.cpp +++ b/src/tests/interop/metrics/extraction_metrics_test.cpp @@ -11,6 +11,7 @@ #include #include #include "inc/extraction_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::io; using namespace illumina::interop; @@ -31,32 +32,41 @@ TYPED_TEST_CASE(extraction_metrics_test, Formats); */ TYPED_TEST(extraction_metrics_test, test_read_write) { + const float tol = 1e-7f; EXPECT_EQ(TypeParam::actual_metric_set.version(), TypeParam::VERSION); EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); EXPECT_EQ(TypeParam::actual_metric_set.max_cycle(), TypeParam::expected_metric_set.max_cycle()); - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); - EXPECT_EQ(itExpected->dateTime(), itActual->dateTime()); + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->date_time(), it_actual->date_time()); - std::time_t t = static_cast(itActual->dateTime()); + std::time_t t = static_cast(it_actual->date_time()); std::tm* tm = std::localtime(&t); EXPECT_NE(tm, static_cast(0)); - - for(size_t i=0;ichannel_count(), it_actual->channel_count()); + for(size_t i=0;ichannel_count();i++) { - EXPECT_EQ(itExpected->max_intensity(i), itActual->max_intensity(i)); - EXPECT_NEAR(itExpected->focusScore(i), itActual->focusScore(i), 1e-7); + EXPECT_EQ(it_expected->max_intensity(i), it_actual->max_intensity(i)); + EXPECT_NEAR(it_expected->focus_score(i), it_actual->focus_score(i), tol); } } } +TEST(run_metrics_extraction_test, test_is_group_empty) +{ + run_metrics metrics; + EXPECT_TRUE(metrics.is_group_empty(constants::Extraction)); + std::istringstream fin(extraction_v2::binary_data()); + io::read_metrics(fin, metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::Extraction)); +} #define FIXTURE extraction_metrics_test /** diff --git a/src/tests/interop/metrics/image_metrics_test.cpp b/src/tests/interop/metrics/image_metrics_test.cpp index e80370df8..f0395468a 100644 --- a/src/tests/interop/metrics/image_metrics_test.cpp +++ b/src/tests/interop/metrics/image_metrics_test.cpp @@ -11,8 +11,10 @@ #include #include #include "inc/image_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::io; +using namespace illumina::interop; using namespace illumina::interop::unittest; @@ -36,24 +38,32 @@ TYPED_TEST(image_metrics_test, test_read_write) { EXPECT_EQ(TypeParam::actual_metric_set.version(), TypeParam::VERSION); EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); - EXPECT_EQ(TypeParam::actual_metric_set.channelCount(), TypeParam::expected_metric_set.channelCount()); + EXPECT_EQ(TypeParam::actual_metric_set.channel_count(), TypeParam::expected_metric_set.channel_count()); EXPECT_EQ(TypeParam::actual_metric_set.max_cycle(), TypeParam::expected_metric_set.max_cycle()); - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); - EXPECT_EQ(itExpected->channelCount(), itActual->channelCount()); - for(size_t i=0;ichannelCount(), itActual->channelCount());i++) + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->channel_count(), it_actual->channel_count()); + for(size_t i=0;ichannel_count(), it_actual->channel_count());i++) { - EXPECT_EQ(itExpected->minContrast(i), itActual->minContrast(i)); - EXPECT_EQ(itExpected->maxContrast(i), itActual->maxContrast(i)); + EXPECT_EQ(it_expected->min_contrast(i), it_actual->min_contrast(i)); + EXPECT_EQ(it_expected->max_contrast(i), it_actual->max_contrast(i)); } } } +TEST(run_metrics_image_test, test_is_group_empty) +{ + run_metrics metrics; + EXPECT_TRUE(metrics.is_group_empty(constants::Image)); + std::istringstream fin(image_v1::binary_data()); + io::read_metrics(fin, metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::Image)); +} #define FIXTURE image_metrics_test /** * @class illumina::interop::model::metrics::image_metric diff --git a/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h b/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h index c9e9ede90..f6ef3b701 100644 --- a/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h +++ b/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h @@ -33,24 +33,24 @@ namespace illumina{ namespace interop { namespace unittest { typedef metric_t::uint_t uint_t; typedef metric_t::ushort_t ushort_t; std::vector metrics; - ushort_t correctedIntAll1[] = {1213, 966, 960, 1095}; - ushort_t correctedIntCalled1[] = {4070, 4074, 4029, 3972}; - uint_t calledCounts1[] = {0, 698433, 548189, 548712, 646638}; + ushort_t corrected_int_all1[] = {1213, 966, 960, 1095}; + ushort_t corrected_int_called1[] = {4070, 4074, 4029, 3972}; + uint_t called_counts1[] = {0, 698433, 548189, 548712, 646638}; metrics.push_back( - metric_t(1, 1104, 25, 1063, 11.9458876f, to_vector(correctedIntCalled1), to_vector(correctedIntAll1), - to_vector(calledCounts1))); - ushort_t correctedIntAll2[] = {1558, 1151, 1158, 1293}; - uint_t calledCounts2[] = {10938, 733661, 537957, 543912, 615504}; - ushort_t correctedIntCalled2[] = {5013, 4983, 4915, 4932}; + metric_t(1, 1104, 25, 1063, 11.9458876f, to_vector(corrected_int_called1), to_vector(corrected_int_all1), + to_vector(called_counts1))); + ushort_t corrected_int_all2[] = {1558, 1151, 1158, 1293}; + uint_t called_counts2[] = {10938, 733661, 537957, 543912, 615504}; + ushort_t corrected_int_called2[] = {5013, 4983, 4915, 4932}; metrics.push_back( - metric_t(1, 1104, 1, 1295, 13.3051805f, to_vector(correctedIntCalled2), to_vector(correctedIntAll2), - to_vector(calledCounts2))); - ushort_t correctedIntAll3[] = {1171, 932, 912, 1069}; - uint_t calledCounts3[] = {0, 706987, 556441, 556067, 653959}; - ushort_t correctedIntCalled3[] = {3931, 3931, 3923, 3878}; + metric_t(1, 1104, 1, 1295, 13.3051805f, to_vector(corrected_int_called2), to_vector(corrected_int_all2), + to_vector(called_counts2))); + ushort_t corrected_int_all3[] = {1171, 932, 912, 1069}; + uint_t called_counts3[] = {0, 706987, 556441, 556067, 653959}; + ushort_t corrected_int_called3[] = {3931, 3931, 3923, 3878}; metrics.push_back( - metric_t(1, 1105, 25, 1025, 11.7396259f, to_vector(correctedIntCalled3), to_vector(correctedIntAll3), - to_vector(calledCounts3))); + metric_t(1, 1105, 25, 1025, 11.7396259f, to_vector(corrected_int_called3), to_vector(corrected_int_all3), + to_vector(called_counts3))); return metrics; } /** Get the expected metric set header @@ -120,18 +120,18 @@ namespace illumina{ namespace interop { namespace unittest { typedef metric_t::uint_t uint_t; typedef metric_t::ushort_t ushort_t; std::vector metrics; - uint_t calledCounts1[] = {52, 1049523, 654071, 500476, 982989}; - ushort_t correctedIntCalled1[] = {245, 252, 61, 235}; - //expected_metrics.push_back(metric_type(7, 1114, 1, to_vector(correctedIntCalled1), to_vector(calledCounts1))); - metrics.push_back(metric_t(7, 1114, 1, (correctedIntCalled1), (calledCounts1))); - uint_t calledCounts2[] = {0, 1063708, 582243, 588028, 953132}; - ushort_t correctedIntCalled2[] = {232, 257, 68, 228}; - //expected_metrics.push_back(metric_type(7, 1114, 2, to_vector(correctedIntCalled2), to_vector(calledCounts2))); - metrics.push_back(metric_t(7, 1114, 2, (correctedIntCalled2), (calledCounts2))); - uint_t calledCounts3[] = {0, 1022928, 617523, 594836, 951825}; - ushort_t correctedIntCalled3[] = {227, 268, 68, 229}; - metrics.push_back(metric_t(7, 1114, 3, (correctedIntCalled3), (calledCounts3))); - //expected_metrics.push_back(metric_type(7, 1114, 3, to_vector(correctedIntCalled3), to_vector(calledCounts3))); + uint_t called_counts1[] = {52, 1049523, 654071, 500476, 982989}; + ushort_t corrected_int_called1[] = {245, 252, 61, 235}; + //expected_metrics.push_back(metric_type(7, 1114, 1, to_vector(corrected_int_called1), to_vector(called_counts1))); + metrics.push_back(metric_t(7, 1114, 1, (corrected_int_called1), (called_counts1))); + uint_t called_counts2[] = {0, 1063708, 582243, 588028, 953132}; + ushort_t corrected_int_called2[] = {232, 257, 68, 228}; + //expected_metrics.push_back(metric_type(7, 1114, 2, to_vector(corrected_int_called2), to_vector(called_counts2))); + metrics.push_back(metric_t(7, 1114, 2, (corrected_int_called2), (called_counts2))); + uint_t called_counts3[] = {0, 1022928, 617523, 594836, 951825}; + ushort_t corrected_int_called3[] = {227, 268, 68, 229}; + metrics.push_back(metric_t(7, 1114, 3, (corrected_int_called3), (called_counts3))); + //expected_metrics.push_back(metric_type(7, 1114, 3, to_vector(corrected_int_called3), to_vector(called_counts3))); return metrics; } /** Get the expected metric set header diff --git a/src/tests/interop/metrics/inc/error_metrics_test.h b/src/tests/interop/metrics/inc/error_metrics_test.h index f450307a5..020414431 100644 --- a/src/tests/interop/metrics/inc/error_metrics_test.h +++ b/src/tests/interop/metrics/inc/error_metrics_test.h @@ -7,95 +7,140 @@ * @copyright GNU Public License. */ #pragma once + #include #include "metric_test.h" #include "interop/model/metrics/error_metric.h" #include "interop/model/summary/run_summary.h" -namespace illumina{ namespace interop { namespace unittest { - -/** This test writes three records of an InterOp files, then reads them back in and compares - * each value to ensure they did not change. - * - * @note Version 3 - */ -struct error_v3 : metric_test +namespace illumina { namespace interop { namespace unittest { - /** Build the expected metric set - * - * @return vector of metrics - */ - static std::vector metrics() - { - std::vector expected_metrics; - - expected_metrics.push_back(metric_t(7, 1114, 1, 0.450100899f)); - expected_metrics.push_back(metric_t(7, 1114, 2, 0.900201797f)); - expected_metrics.push_back(metric_t(7, 1114, 3, 0.465621591f)); - - return expected_metrics; - } - /** Get the expected metric set header + /** This generator creates an expected metric set and the corresponding binary data * - * @return expected metric set header + * @note Version 3 */ - static header_t header() + struct error_metric_v3 : metric_test { - return header_t(); - } - /** Get the expected binary data - * - * @return binary data string - */ - static std::string binary_data() - { - const int tmp[] = {3,30,7,0,90,4,1,0,-96,115,-26,62,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,7,0,90,4,2,0,-96,115,102,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,7,0,90,4,3,0,-12,101,-18,62,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - }; - return to_string(tmp); - } - /** Get number lanes in data - * - * @return 8 lanes - */ - static model::metric_base::base_metric::uint_t lane_count(){return 8;} - /** Get reads describing data - * - * @return reads vector - */ - static std::vector reads() - { - std::vector reads(1, model::run::read_info(1, 1, 3, false)); - return reads; - } - /** Get the summary for these metrics + //typedef model::metrics::error_metric metric_t; + //typedef model::metric_base::metric_set< metric_t > metric_set_t; + /** Create the expected metric set + * + * @param metrics destination metric set + */ + static void create_expected(metric_set_t& metrics) + { + metrics = metric_set_t(3); + metrics.insert(metric_t(7, 1114, 1, 0.450100899f)); + metrics.insert(metric_t(7, 1114, 2, 0.900201797f)); + metrics.insert(metric_t(7, 1114, 3, 0.465621591f)); + } + /** Get the expected binary data + * + * @return binary data string + */ + static std::string binary_data() + { + const int tmp[] = {3, 30, 7, 0, 90, 4, 1, 0, -96, 115, -26, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 7, 0, 90, 4, 2, 0, -96, 115, 102, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 90, 4, 3, 0, -12, 101, -18, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + return to_string(tmp); + } + }; + + /** This test writes three records of an InterOp files, then reads them back in and compares + * each value to ensure they did not change. * - * @return run summary + * @note Version 3 */ - static model::summary::run_summary summary() + struct error_v3 : metric_test { - std::vector read_infos = reads(); - model::summary::run_summary summary(read_infos.begin(), read_infos.end(), 1); - summary[0][0].lane(7); - summary[0][0].error_rate()=model::summary::metric_stat(0.67515134811401367f, 0, 0.67515134811401367f); - summary[0][0].error_rate_35()=model::summary::metric_stat(0.0f, 0, 0.0f); - summary[0][0].error_rate_50()=model::summary::metric_stat(00.0f, 0, 0.0f); - summary[0][0].error_rate_75()=model::summary::metric_stat(0.0f, 0, 0.0f); - summary[0][0].error_rate_100()=model::summary::metric_stat(0.0f, 0, 0.0f); - summary[0][0].cycle_state().error_cycle_range(model::run::cycle_range(3, 3)); - summary[0].summary().error_rate(0.67515134811401367f); - summary.total_summary().error_rate(0.67515134811401367f); - summary.nonindex_summary().error_rate(0.67515134811401367f); - summary[0][0].tile_count(1); - summary.cycle_state().error_cycle_range(model::run::cycle_range(3, 3)); - return summary; - } -}; + /** Build the expected metric set + * + * @return vector of metrics + */ + static std::vector metrics() + { + std::vector expected_metrics; + + + expected_metrics.push_back(metric_t(7, 1114, 1, 0.450100899f)); + expected_metrics.push_back(metric_t(7, 1114, 2, 0.900201797f)); + expected_metrics.push_back(metric_t(7, 1114, 3, 0.465621591f)); + + return expected_metrics; + } + + /** Get the expected metric set header + * + * @return expected metric set header + */ + static header_t header() + { + return header_t(); + } + + /** Get the expected binary data + * + * @return binary data string + */ + static std::string binary_data() + { + const int tmp[] = {3, 30, 7, 0, 90, 4, 1, 0, -96, 115, -26, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 7, 0, 90, 4, 2, 0, -96, 115, 102, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 90, 4, 3, 0, -12, 101, -18, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + return to_string(tmp); + } + + /** Get number lanes in data + * + * @return 8 lanes + */ + static model::metric_base::base_metric::uint_t lane_count() + { return 8; } + + /** Get reads describing data + * + * @return reads vector + */ + static std::vector reads() + { + std::vector reads(1, model::run::read_info(1, 1, 3, false)); + return reads; + } + + /** Get the summary for these metrics + * + * @return run summary + */ + static model::summary::run_summary summary() + { + std::vector read_infos = reads(); + model::summary::run_summary summary(read_infos.begin(), read_infos.end(), 1); + summary[0][0].lane(7); + summary[0][0].error_rate(model::summary::metric_stat(0.67515134811401367f, 0, 0.67515134811401367f)); + summary[0][0].error_rate_35(model::summary::metric_stat(0.0f, 0, 0.0f)); + summary[0][0].error_rate_50(model::summary::metric_stat(00.0f, 0, 0.0f)); + summary[0][0].error_rate_75(model::summary::metric_stat(0.0f, 0, 0.0f)); + summary[0][0].error_rate_100(model::summary::metric_stat(0.0f, 0, 0.0f)); + summary[0][0].cycle_state().error_cycle_range(model::run::cycle_range(3, 3)); + summary[0].summary().error_rate(0.67515134811401367f); + summary.total_summary().error_rate(0.67515134811401367f); + summary.nonindex_summary().error_rate(0.67515134811401367f); + summary[0][0].tile_count(1); + summary.cycle_state().error_cycle_range(model::run::cycle_range(3, 3)); + return summary; + } + }; /** Interface between fixtures and Google Test */ -template -struct error_metrics_test : public ::testing::Test, public TestSetup{}; + template + struct error_metrics_test : public ::testing::Test, public TestSetup + { + }; }}} diff --git a/src/tests/interop/metrics/inc/extraction_metrics_test.h b/src/tests/interop/metrics/inc/extraction_metrics_test.h index d36f6ecaa..8517ad57e 100644 --- a/src/tests/interop/metrics/inc/extraction_metrics_test.h +++ b/src/tests/interop/metrics/inc/extraction_metrics_test.h @@ -87,7 +87,7 @@ namespace illumina{ namespace interop { namespace unittest { std::vector read_infos = reads(); model::summary::run_summary summary(read_infos.begin(), read_infos.end(), 1); summary[0][0].lane(7); - summary[0][0].first_cycle_intensity()=model::summary::metric_stat(321, 24.75883674621582f, 312); + summary[0][0].first_cycle_intensity(model::summary::metric_stat(321, 24.75883674621582f, 312)); summary[0][0].cycle_state().extracted_cycle_range(model::run::cycle_range(1, 1)); summary[0].summary().first_cycle_intensity(321); summary.total_summary().first_cycle_intensity(321); diff --git a/src/tests/interop/metrics/inc/image_metrics_test.h b/src/tests/interop/metrics/inc/image_metrics_test.h index b7b79e66a..bffd43f92 100644 --- a/src/tests/interop/metrics/inc/image_metrics_test.h +++ b/src/tests/interop/metrics/inc/image_metrics_test.h @@ -25,7 +25,7 @@ namespace illumina{ namespace interop { namespace unittest { { enum{ /** Do not check the expected binary data */ - disable_binary_data=true // TODO: Move this to template? + disable_binary_data=true // The order here could change }; /** Build the expected metric set * @@ -99,7 +99,7 @@ namespace illumina{ namespace interop { namespace unittest { typedef metric_t::ushort_t ushort_t; - const ushort_t channel_count = header().channelCount(); + const ushort_t channel_count = header().channel_count(); const ushort_t min_contrast1[] = {231, 207}; const ushort_t min_contrast2[] = {229, 205}; const ushort_t min_contrast3[] = {231, 222}; diff --git a/src/tests/interop/metrics/inc/metric_generator.h b/src/tests/interop/metrics/inc/metric_generator.h new file mode 100644 index 000000000..848393740 --- /dev/null +++ b/src/tests/interop/metrics/inc/metric_generator.h @@ -0,0 +1,165 @@ +/** Classes that generated actual and expected metric sets for testing + * + * @file + * @date 8/10/16. + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include "interop/io/metric_file_stream.h" +#include "src/tests/interop/inc/regression_test_data.h" + +namespace illumina{ namespace interop { namespace unittest { + + /** Generate the actual metric set by reading in from hardcoded binary buffer + * + * The expected metric set is provided by the generator. + */ + template + class hardcoded_metric_generator : public abstract_generator< typename Gen::metric_set_t > + { + typedef typename Gen::metric_set_t metric_set_t; + public: + /** Generate the expected and actual metric sets + * + * @param expected expected metric set + * @param actual actual metric set + */ + bool generate(metric_set_t& expected, metric_set_t& actual) + { + actual.clear(); + Gen::create_expected(expected); + try + { + std::istringstream fin(Gen::binary_data()); + illumina::interop::io::read_metrics(fin, actual); + } + catch (const std::exception &) { } + return true; + } + }; + /** Generate the actual metric set writing out the expected and reading it back in again + * + * The expected metric set is provided by the generator. + */ + template + class write_read_metric_generator : public abstract_generator< typename Gen::metric_set_t > + { + typedef typename Gen::metric_set_t metric_set_t; + public: + /** Generate the expected and actual metric sets + * + * @param expected expected metric set + * @param actual actual metric set + */ + bool generate(metric_set_t& expected, metric_set_t& actual) + { + actual.clear(); + Gen::create_expected(expected); + std::ostringstream fout; + try + { + illumina::interop::io::write_metrics(fout, expected); + } + catch (const std::exception &) { } + try + { + std::istringstream fin(fout.str()); + illumina::interop::io::read_metrics(fin, actual); + } + catch (const std::exception &) { } + return true; + } + }; + /** Generate the actual metric set writing out the expected and reading it back in again + * + * The expected metric set is provided by the generator. + */ + template + class regression_test_metric_generator : public abstract_generator< MetricSet > + { + typedef MetricSet metric_set_t; + public: + typedef abstract_generator< MetricSet >* parent_type; + public: + regression_test_metric_generator(const std::string& test_dir) : m_test_dir(test_dir) + { + } + void operator()(const std::string& name)const + { + m_run_folder = name; + } + /** Generate the expected and actual metric sets + * + * @param expected expected metric set + * @param actual actual metric set + */ + bool generate(metric_set_t& expected, metric_set_t& actual) + { + const regression_test_data& data = regression_test_data::instance(); + const std::string baseline_file = io::combine(io::combine(data.baseline(), m_test_dir), io::basename(m_run_folder)); + if(!data.rebaseline()) + { + if(io::is_file_readable(baseline_file)) + { + expected.clear(); + actual.clear(); + illumina::interop::io::read_interop(baseline_file, expected); + illumina::interop::io::read_interop(m_run_folder, actual); + return true; + } + else EXPECT_TRUE(false) << "Failed to find baseline: " << baseline_file; + } + else + { + std::cout << "[ ] Rebaseline: " << io::basename(m_run_folder) << std::endl; + actual.clear(); + illumina::interop::io::read_interop(m_run_folder, actual); + try + { + illumina::interop::io::write_interop(baseline_file, actual); + } + catch(const std::exception&) + { + EXPECT_TRUE(false)<< "Failed to write baseline: " << baseline_file; + } + } + return false; + } + private: + mutable std::string m_run_folder; + std::string m_test_dir; + }; + + /** Generate the actual binary data by writing out the expected metric set + * + * The expected binary data is provided by the generator. + */ + template + class write_metric_generator : public abstract_generator< std::string > + { + typedef typename Gen::metric_set_t metric_set_t; + public: + /** Generate the expected and actual metric sets + * + * @param expected expected binary buffer + * @param actual actual binary buffer + */ + bool generate(std::string& expected, std::string& actual) + { + expected = Gen::binary_data(); + + metric_set_t metrics; + Gen::create_expected(metrics); + std::ostringstream fout; + try + { + illumina::interop::io::write_metrics(fout, metrics); + } + catch (const std::exception &) { } + actual = fout.str(); + return true; + } + }; + +}}} diff --git a/src/tests/interop/metrics/inc/metric_stream_tests.hpp b/src/tests/interop/metrics/inc/metric_stream_tests.hpp new file mode 100644 index 000000000..19204afb4 --- /dev/null +++ b/src/tests/interop/metrics/inc/metric_stream_tests.hpp @@ -0,0 +1,122 @@ +/** Unit tests for various failure conditions handled by exceptions. + * + * + * @file + * @date 10/6/2015 + * @version 1.0 + * @copyright GNU Public License. + */ + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4127) // MSVC warns about using constants in conditional statements, for template constants +#endif + +/** Wrapper for X-Macro FIXTURE + * + */ +#define WRAPPED_TEST(FIXTURE, FUNC) TEST_P(FIXTURE, FUNC) + + +/** + * Confirm binary write matches expected binary data + */ +WRAPPED_TEST(FIXTURE, test_write_read_binary_data) +{ +# ifdef INTEROP_BINARY_DATA_SIZE_CHECK + return; +# endif + EXPECT_EQ(expected.size(), actual.size()); +# ifdef INTEROP_BINARY_DATA_CHECK + return; +# endif + for(::uint32_t i=0;i10); +} +/** Confirm incomplete_file_exception is thrown for a mostly complete file + */ +WRAPPED_TEST(FIXTURE, test_hardcoded_incomplete_file_exception_last_metric) +{ + std::istringstream fin(expected.substr(0,expected.length()-4)); + metric_set_t metrics; + EXPECT_THROW(illumina::interop::io::read_metrics(fin, metrics), incomplete_file_exception); +} +/** Confirm bad_format_exception is thrown when record size is incorrect + */ +WRAPPED_TEST(FIXTURE, test_hardcoded_incorrect_record_size) +{ +# ifdef INTEROP_SKIP_RECORD_SIZE_CHECK + return; +# endif + std::string tmp = std::string(expected); + tmp[1] = 0; + std::istringstream fin(tmp); + metric_set_t metrics; + EXPECT_THROW(illumina::interop::io::read_metrics(fin, metrics), bad_format_exception); +} +/** Confirm file_not_found_exception is thrown when a file is not found + */ +WRAPPED_TEST(FIXTURE, test_hardcoded_file_not_found) +{ + metric_set_t metrics; + EXPECT_THROW(illumina::interop::io::read_interop("/NO/FILE/EXISTS", metrics), file_not_found_exception); +} +/** Confirm reading from good data does not throw an exception + */ +WRAPPED_TEST(FIXTURE, test_hardcoded_read) +{ + std::string tmp = std::string(expected); + std::istringstream fin(tmp); + metric_set_t metrics; + EXPECT_NO_THROW(illumina::interop::io::read_metrics(fin, metrics)); +} +/** Confirm the clear function works + */ +WRAPPED_TEST(FIXTURE, test_clear) +{ + std::string tmp = std::string(expected); + std::istringstream fin(tmp); + metric_set_t metrics; + EXPECT_NO_THROW(illumina::interop::io::read_metrics(fin, metrics)); + size_t cnt = metrics.size(); + metrics.clear(); + std::istringstream fin2(tmp); + EXPECT_NO_THROW(illumina::interop::io::read_metrics(fin2, metrics)); + EXPECT_EQ(cnt, metrics.size()); + +} + +#ifdef _MSC_VER +#pragma warning(pop) +#endif diff --git a/src/tests/interop/metrics/inc/metric_test.h b/src/tests/interop/metrics/inc/metric_test.h index d1561aa99..9c28090a2 100644 --- a/src/tests/interop/metrics/inc/metric_test.h +++ b/src/tests/interop/metrics/inc/metric_test.h @@ -39,7 +39,7 @@ namespace illumina{ namespace interop { namespace unittest { class metric_test { public: - enum + enum Checks { /** Version of the format */ VERSION=Version, diff --git a/src/tests/interop/metrics/inc/q_metrics_test.h b/src/tests/interop/metrics/inc/q_metrics_test.h index 211e2319a..168e49c97 100644 --- a/src/tests/interop/metrics/inc/q_metrics_test.h +++ b/src/tests/interop/metrics/inc/q_metrics_test.h @@ -270,7 +270,7 @@ namespace illumina{ namespace interop { namespace unittest { * @return expected metric set header */ static header_t header() - {; + { typedef header_t::qscore_bin_vector_type qscore_bin_vector_type; typedef header_t::bin_t bin_t; typedef bin_t::bin_type ushort_t; diff --git a/src/tests/interop/metrics/inc/summary_fixture.h b/src/tests/interop/metrics/inc/summary_fixture.h index 43dcb0f11..0567b53ba 100644 --- a/src/tests/interop/metrics/inc/summary_fixture.h +++ b/src/tests/interop/metrics/inc/summary_fixture.h @@ -17,7 +17,10 @@ namespace illumina{ namespace interop { namespace unittest { template struct summary_fixture { - /** Constructor */ + /** Constructor + * + * @param reads list of reads + */ summary_fixture(std::vector reads = Gen::reads()) : expected(Gen::summary()), actual(reads.begin(), reads.end(), Gen::lane_count()) @@ -28,7 +31,7 @@ namespace illumina{ namespace interop { namespace unittest { model::run::info run_info("XX", "", 1, - model::run::flowcell_layout(Gen::lane_count()), + model::run::flowcell_layout(Gen::lane_count(), 2, 4, 99, 6, 6), channels, model::run::image_dimensions(), reads); diff --git a/src/tests/interop/metrics/inc/tile_metrics_test.h b/src/tests/interop/metrics/inc/tile_metrics_test.h index 39c00d91d..0fd04bf98 100644 --- a/src/tests/interop/metrics/inc/tile_metrics_test.h +++ b/src/tests/interop/metrics/inc/tile_metrics_test.h @@ -128,19 +128,19 @@ namespace illumina{ namespace interop { namespace unittest { summary[read][0].tile_count(3); summary[read][0].reads_pf(9738715); summary[read][0].reads(19412848); - summary[read][0].density() = model::summary::metric_stat(2355119.25f, 0, 2355119.25f); - summary[read][0].density_pf() = model::summary::metric_stat(1181477.125f, 27380.955078125f, 1174757.75f); - summary[read][0].cluster_count() = model::summary::metric_stat(6470949.5f, 0, 6470949); - summary[read][0].cluster_count_pf() = model::summary::metric_stat(3246238.25f, 75232.1640625f, 3227776); - summary[read][0].percent_pf() = model::summary::metric_stat(50.166339874267578f, 1.1626163721084595f, 49.881031036376953f); + summary[read][0].density(model::summary::metric_stat(2355119.25f, 0, 2355119.25f)); + summary[read][0].density_pf(model::summary::metric_stat(1181477.125f, 27380.955078125f, 1174757.75f)); + summary[read][0].cluster_count(model::summary::metric_stat(6470949.5f, 0, 6470949)); + summary[read][0].cluster_count_pf(model::summary::metric_stat(3246238.25f, 75232.1640625f, 3227776)); + summary[read][0].percent_pf(model::summary::metric_stat(50.166339874267578f, 1.1626163721084595f, 49.881031036376953f)); } - summary[0][0].phasing() = model::summary::metric_stat(0.10935487598180771f, 0.026172075420618057f, 0.11908555030822754f); - summary[0][0].prephasing() = model::summary::metric_stat(0.1159147247672081f, 0.021491257473826408f, 0.11990892142057419f); - summary[0][0].percent_aligned() = model::summary::metric_stat(2.5763518810272217f, 0.074578315019607544f, 2.6163086891174316f); + summary[0][0].phasing(model::summary::metric_stat(0.10935487598180771f, 0.026172075420618057f, 0.11908555030822754f)); + summary[0][0].prephasing(model::summary::metric_stat(0.1159147247672081f, 0.021491257473826408f, 0.11990892142057419f)); + summary[0][0].percent_aligned(model::summary::metric_stat(2.5763518810272217f, 0.074578315019607544f, 2.6163086891174316f)); - summary[1][0].phasing() = model::summary::metric_stat(0.079711258411407471f, 0, 0.079711258411407471f); - summary[1][0].prephasing() = model::summary::metric_stat(0.11990892142057419f, 0, 0.11990892142057419f); - summary[1][0].percent_aligned() = model::summary::metric_stat(2.6163086891174316, 0, 2.6163086891174316f); + summary[1][0].phasing(model::summary::metric_stat(0.079711258411407471f, 0, 0.079711258411407471f)); + summary[1][0].prephasing(model::summary::metric_stat(0.11990892142057419f, 0, 0.11990892142057419f)); + summary[1][0].percent_aligned(model::summary::metric_stat(2.6163086891174316, 0, 2.6163086891174316f)); summary[0].summary().percent_aligned(2.5763518810272217f); summary[1].summary().percent_aligned(2.6163086891174316f); diff --git a/src/tests/interop/metrics/index_metrics_test.cpp b/src/tests/interop/metrics/index_metrics_test.cpp index b04b63823..8446f4fec 100644 --- a/src/tests/interop/metrics/index_metrics_test.cpp +++ b/src/tests/interop/metrics/index_metrics_test.cpp @@ -10,8 +10,10 @@ #include #include #include "inc/index_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::io; +using namespace illumina::interop; using namespace illumina::interop::unittest; typedef ::testing::Types< @@ -30,24 +32,32 @@ TYPED_TEST(index_metrics_test, test_read_write) EXPECT_EQ(TypeParam::actual_metric_set.version(), TypeParam::VERSION); EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->read(), itActual->read()); - EXPECT_EQ(itExpected->size(), itActual->size()); - for(size_t i=0;isize(), itActual->size());i++) + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->read(), it_actual->read()); + EXPECT_EQ(it_expected->size(), it_actual->size()); + for(size_t i=0;isize(), it_actual->size());i++) { - EXPECT_EQ(itExpected->indices(i).index_seq(), itActual->indices(i).index_seq()); - EXPECT_EQ(itExpected->indices(i).sample_id(), itActual->indices(i).sample_id()); - EXPECT_EQ(itExpected->indices(i).sample_proj(), itActual->indices(i).sample_proj()); - EXPECT_EQ(itExpected->indices(i).count(), itActual->indices(i).count()); + EXPECT_EQ(it_expected->indices(i).index_seq(), it_actual->indices(i).index_seq()); + EXPECT_EQ(it_expected->indices(i).sample_id(), it_actual->indices(i).sample_id()); + EXPECT_EQ(it_expected->indices(i).sample_proj(), it_actual->indices(i).sample_proj()); + EXPECT_EQ(it_expected->indices(i).count(), it_actual->indices(i).count()); } } } +TEST(run_metrics_index_test, test_is_group_empty) +{ + run_metrics metrics; + EXPECT_TRUE(metrics.is_group_empty(constants::Index)); + std::istringstream fin(index_v1::binary_data()); + io::read_metrics(fin, metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::Index)); +} #define FIXTURE index_metrics_test /** * @class illumina::interop::model::metrics::index_metric diff --git a/src/tests/interop/metrics/q_by_lane_metric_test.cpp b/src/tests/interop/metrics/q_by_lane_metric_test.cpp index 8be8de38f..127538e7f 100644 --- a/src/tests/interop/metrics/q_by_lane_metric_test.cpp +++ b/src/tests/interop/metrics/q_by_lane_metric_test.cpp @@ -11,6 +11,7 @@ #include "interop/model/metrics/q_by_lane_metric.h" #include "inc/q_metrics_test.h" #include "interop/logic/metric/q_metric.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop; using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; @@ -46,18 +47,28 @@ TEST(q_by_lane_metrics_test, test_convert_write_read) EXPECT_EQ(actual_metric_set.version(), expected_metric_set.version()); EXPECT_EQ(actual_metric_set.max_cycle(), expected_metric_set.max_cycle()); ASSERT_EQ(actual_metric_set.size(), expected_metric_set.size()); - for(metric_set::const_iterator itExpected=expected_metric_set.begin(), - itActual = actual_metric_set.begin(); - itExpected != expected_metric_set.end(); - itExpected++,itActual++) + for(metric_set::const_iterator it_expected=expected_metric_set.begin(), + it_actual = actual_metric_set.begin(); + it_expected != expected_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); - EXPECT_EQ(itExpected->size(), itActual->size()); - for(size_t i=0;isize(), itActual->size());i++) + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->size(), it_actual->size()); + for(size_t i=0;isize(), it_actual->size());i++) { - EXPECT_EQ(itExpected->qscoreHist(i), itActual->qscoreHist(i)); + EXPECT_EQ(it_expected->qscore_hist(i), it_actual->qscore_hist(i)); } } } + +TEST(run_metrics_q_by_lane_test, test_is_group_empty) +{ + run_metrics metrics; + EXPECT_TRUE(metrics.is_group_empty(constants::QByLane)); + std::istringstream fin(q_v4::binary_data()); + io::read_metrics(fin, metrics.get_set()); + logic::metric::create_q_metrics_by_lane(metrics.get_set(), metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::QByLane)); +} \ No newline at end of file diff --git a/src/tests/interop/metrics/q_collapsed_metrics_test.cpp b/src/tests/interop/metrics/q_collapsed_metrics_test.cpp index 50f6a5da9..a91b90fc0 100644 --- a/src/tests/interop/metrics/q_collapsed_metrics_test.cpp +++ b/src/tests/interop/metrics/q_collapsed_metrics_test.cpp @@ -11,6 +11,7 @@ #include "interop/logic/metric/q_metric.h" #include "inc/q_collapsed_metrics_test.h" #include "inc/q_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; @@ -40,13 +41,13 @@ TYPED_TEST(q_collapsed_metrics_test, test_read_write) EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); EXPECT_EQ(TypeParam::actual_metric_set.max_cycle(), TypeParam::expected_metric_set.max_cycle()); - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); } } @@ -73,20 +74,30 @@ TEST(q_collapsed_metrics_test, test_convert_write_read) EXPECT_EQ(actual_metric_set.version(), expected_metric_set.version()); EXPECT_EQ(actual_metric_set.max_cycle(), expected_metric_set.max_cycle()); ASSERT_EQ(actual_metric_set.size(), expected_metric_set.size()); - for(metric_set::const_iterator itExpected=expected_metric_set.begin(), - itActual = actual_metric_set.begin(); - itExpected != expected_metric_set.end(); - itExpected++,itActual++) + for(metric_set::const_iterator it_expected=expected_metric_set.begin(), + it_actual = actual_metric_set.begin(); + it_expected != expected_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); - EXPECT_EQ(itExpected->q20(), itActual->q20()); - EXPECT_EQ(itExpected->q30(), itActual->q30()); - EXPECT_EQ(itExpected->median_qscore(), itActual->median_qscore()); + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->q20(), it_actual->q20()); + EXPECT_EQ(it_expected->q30(), it_actual->q30()); + EXPECT_EQ(it_expected->median_qscore(), it_actual->median_qscore()); } } +TEST(run_metrics_q_collapsed_test, test_is_group_empty) +{ + run_metrics metrics; + EXPECT_TRUE(metrics.is_group_empty(constants::QCollapsed)); + std::istringstream fin(q_v4::binary_data()); + io::read_metrics(fin, metrics.get_set()); + logic::metric::create_collapse_q_metrics(metrics.get_set(), metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::QCollapsed)); +} + #define FIXTURE q_collapsed_metrics_test /** * @class illumina::interop::model::metrics::q_collapsed_metric diff --git a/src/tests/interop/metrics/q_metrics_test.cpp b/src/tests/interop/metrics/q_metrics_test.cpp index 35f8a1741..70d775599 100644 --- a/src/tests/interop/metrics/q_metrics_test.cpp +++ b/src/tests/interop/metrics/q_metrics_test.cpp @@ -2,7 +2,6 @@ * * * @file - * * @date 8/25/2015 * @version 1.0 * @copyright GNU Public License. @@ -13,6 +12,7 @@ #include #include "interop/logic/metric/q_metric.h" #include "inc/q_metrics_test.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; using namespace illumina::interop::io; @@ -45,26 +45,26 @@ TYPED_TEST(q_metrics_test, test_read_write) EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); EXPECT_EQ(TypeParam::actual_metric_set.max_cycle(), TypeParam::expected_metric_set.max_cycle()); - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - EXPECT_EQ(itExpected->cycle(), itActual->cycle()); - EXPECT_EQ(itExpected->size(), itActual->size()); - for(size_t i=0;isize(), itActual->size());i++) + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + EXPECT_EQ(it_expected->cycle(), it_actual->cycle()); + EXPECT_EQ(it_expected->size(), it_actual->size()); + for(size_t i=0;isize(), it_actual->size());i++) { - EXPECT_EQ(itExpected->qscoreHist(i), itActual->qscoreHist(i)); + EXPECT_EQ(it_expected->qscore_hist(i), it_actual->qscore_hist(i)); } } EXPECT_EQ(logic::metric::count_q_metric_bins(TypeParam::actual_metric_set),logic::metric::count_q_metric_bins(TypeParam::expected_metric_set)); - EXPECT_EQ(TypeParam::actual_metric_set.binCount(), TypeParam::expected_metric_set.binCount()); - for(size_t i=0;i()); + EXPECT_FALSE(metrics.is_group_empty(constants::Q)); +} + #define FIXTURE q_metrics_test /** * @class illumina::interop::model::metrics::q_metric diff --git a/src/tests/interop/metrics/summary_metrics_test.cpp b/src/tests/interop/metrics/summary_metrics_test.cpp index dfaeac0d3..41b4a34bd 100644 --- a/src/tests/interop/metrics/summary_metrics_test.cpp +++ b/src/tests/interop/metrics/summary_metrics_test.cpp @@ -9,6 +9,7 @@ #include #include +#include "interop/util/math.h" #include "interop/logic/summary/run_summary.h" #include "inc/error_metrics_test.h" #include "inc/extraction_metrics_test.h" @@ -16,7 +17,7 @@ #include "inc/q_metrics_test.h" #include "inc/index_metrics_test.h" #include "inc/summary_fixture.h" -#include "../inc/regression_fixture.h" +#include "src/tests/interop/inc/regression_fixture.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::io; using namespace illumina::interop; @@ -52,41 +53,53 @@ TYPED_TEST_CASE(summary_metrics_test, Formats); EXPECT_EQ(ACTUAL.number(), EXPECTED.number()); \ EXPECT_EQ(ACTUAL.is_index(), EXPECTED.is_index()) -void test_run_summary(const model::summary::run_summary& expected, const model::summary::run_summary& actual) + + +/** Test if calculated run summary matches actual run summary + * + * @todo Add more robust testing (e.g. non-index) + */ +TYPED_TEST(summary_metrics_test, run_summary) { + const model::summary::run_summary& expected_summary=TypeParam::expected; + const model::summary::run_summary& actual_summary=TypeParam::actual; const float tol = 1e-7f; - ASSERT_EQ(actual.size(), expected.size()); - ASSERT_EQ(actual.lane_count(), expected.lane_count()); - EXPECT_NEAR(actual.total_summary().error_rate(), expected.total_summary().error_rate(), tol); - EXPECT_NEAR(actual.total_summary().percent_aligned(), expected.total_summary().percent_aligned(), tol); - EXPECT_NEAR(actual.total_summary().first_cycle_intensity(), expected.total_summary().first_cycle_intensity(), tol); - EXPECT_NEAR(actual.total_summary().percent_gt_q30(), expected.total_summary().percent_gt_q30(), tol); - EXPECT_NEAR(actual.total_summary().yield_g(), expected.total_summary().yield_g(), tol); - EXPECT_NEAR(actual.total_summary().projected_yield_g(), expected.total_summary().projected_yield_g(), tol); + ASSERT_EQ(actual_summary.size(), expected_summary.size()); + ASSERT_EQ(actual_summary.lane_count(), expected_summary.lane_count()); + EXPECT_NEAR(actual_summary.total_summary().error_rate(), expected_summary.total_summary().error_rate(), tol); + EXPECT_NEAR(actual_summary.total_summary().percent_aligned(), expected_summary.total_summary().percent_aligned(), tol); + EXPECT_NEAR(actual_summary.total_summary().first_cycle_intensity(), expected_summary.total_summary().first_cycle_intensity(), tol); + EXPECT_NEAR(actual_summary.total_summary().percent_gt_q30(), expected_summary.total_summary().percent_gt_q30(), tol); + EXPECT_NEAR(actual_summary.total_summary().yield_g(), expected_summary.total_summary().yield_g(), tol); + EXPECT_NEAR(actual_summary.total_summary().projected_yield_g(), expected_summary.total_summary().projected_yield_g(), tol); - EXPECT_NEAR(actual.nonindex_summary().error_rate(), expected.nonindex_summary().error_rate(), tol); - EXPECT_NEAR(actual.nonindex_summary().percent_aligned(), expected.nonindex_summary().percent_aligned(), tol); - EXPECT_NEAR(actual.nonindex_summary().first_cycle_intensity(), expected.nonindex_summary().first_cycle_intensity(), tol); - EXPECT_NEAR(actual.nonindex_summary().percent_gt_q30(), expected.nonindex_summary().percent_gt_q30(), tol); - EXPECT_NEAR(actual.nonindex_summary().yield_g(), expected.nonindex_summary().yield_g(), tol); - EXPECT_NEAR(actual.nonindex_summary().projected_yield_g(), expected.nonindex_summary().projected_yield_g(), tol); + EXPECT_NEAR(actual_summary.nonindex_summary().error_rate(), expected_summary.nonindex_summary().error_rate(), tol); + EXPECT_NEAR(actual_summary.nonindex_summary().percent_aligned(), expected_summary.nonindex_summary().percent_aligned(), tol); + EXPECT_NEAR(actual_summary.nonindex_summary().first_cycle_intensity(), expected_summary.nonindex_summary().first_cycle_intensity(), tol); + EXPECT_NEAR(actual_summary.nonindex_summary().percent_gt_q30(), expected_summary.nonindex_summary().percent_gt_q30(), tol); + EXPECT_NEAR(actual_summary.nonindex_summary().yield_g(), expected_summary.nonindex_summary().yield_g(), tol); + EXPECT_NEAR(actual_summary.nonindex_summary().projected_yield_g(), expected_summary.nonindex_summary().projected_yield_g(), tol); - const model::summary::cycle_state_summary& actual_cycle_summary = actual.cycle_state(); - const model::summary::cycle_state_summary& expected_cycle_summary = expected.cycle_state(); + const model::summary::cycle_state_summary& actual_cycle_summary = actual_summary.cycle_state(); + const model::summary::cycle_state_summary& expected_cycle_summary = expected_summary.cycle_state(); EXPECT_CYCLE_EQ(actual_cycle_summary.extracted_cycle_range(), expected_cycle_summary.extracted_cycle_range()); EXPECT_CYCLE_EQ(actual_cycle_summary.called_cycle_range(), expected_cycle_summary.called_cycle_range()); EXPECT_CYCLE_EQ(actual_cycle_summary.qscored_cycle_range(), expected_cycle_summary.qscored_cycle_range()); EXPECT_CYCLE_EQ(actual_cycle_summary.error_cycle_range(), expected_cycle_summary.error_cycle_range()); } -void test_read_summary(const model::summary::run_summary& expected, const model::summary::run_summary& actual) +/** Test if calculated read summary matches actual read summary + */ +TYPED_TEST(summary_metrics_test, read_summary) { - ASSERT_EQ(actual.size(), expected.size()); + const model::summary::run_summary& expected_summary=TypeParam::expected; + const model::summary::run_summary& actual_summary=TypeParam::actual; + ASSERT_EQ(actual_summary.size(), expected_summary.size()); const float tol = 1e-7f; - for(size_t read=0;read -{ - /** Constructor */ - summary_regression_test() : regression_test_fixture< summary_regression_test, model::summary::run_summary>("summary"){} - /** Populate the actual summary metrics using the given run_metrics - * - * @param actual_metrics run_metrics read in from a run_folder - * @param actual run_summary constructed from the run_metrics - */ - static void populate_actual(model::metrics::run_metrics& actual_metrics, model::summary::run_summary& actual) - { - logic::summary::summarize_run_metrics(actual_metrics, actual); - } -}; - +//struct summary_regression_test : public regression_test_fixture< summary_regression_test, model::summary::run_summary> +//{ +// /** Constructor */ +// summary_regression_test() : regression_test_fixture< summary_regression_test, model::summary::run_summary>("summary"){} +// /** Populate the actual summary metrics using the given run_metrics +// * +// * @param actual_metrics run_metrics read in from a run_folder +// * @param actual run_summary constructed from the run_metrics +// */ +// static void populate_actual(model::metrics::run_metrics& actual_metrics, model::summary::run_summary& actual) +// { +// logic::summary::summarize_run_metrics(actual_metrics, actual); +// } +//}; +/* TEST_P(summary_regression_test, compare_to_baseline) { if(!test) return; @@ -260,3 +256,4 @@ TEST_P(summary_regression_test, compare_to_baseline) INSTANTIATE_TEST_CASE_P(regression_input, summary_regression_test, PersistentValuesIn(regression_test_data::instance().files())); +*/ \ No newline at end of file diff --git a/src/tests/interop/metrics/tile_metrics_test.cpp b/src/tests/interop/metrics/tile_metrics_test.cpp index 67e4d637c..25359fc07 100644 --- a/src/tests/interop/metrics/tile_metrics_test.cpp +++ b/src/tests/interop/metrics/tile_metrics_test.cpp @@ -11,8 +11,9 @@ #include #include #include "inc/tile_metrics_test.h" +#include "interop/util/math.h" #include "interop/util/statistics.h" -#include "interop/util/type_traits.h" +#include "interop/model/run_metrics.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; using namespace illumina::interop::io; @@ -40,27 +41,29 @@ TYPED_TEST(tile_metrics_test, test_read_write) EXPECT_EQ(TypeParam::actual_metric_set.size(), TypeParam::expected_metric_set.size()); const float tol = 1e-7f / 0.01f; - for(typename TypeParam::const_iterator itExpected=TypeParam::expected_metric_set.begin(), itActual = TypeParam::actual_metric_set.begin(); - itExpected != TypeParam::expected_metric_set.end() && itActual != TypeParam::actual_metric_set.end(); - itExpected++,itActual++) + for(typename TypeParam::const_iterator it_expected=TypeParam::expected_metric_set.begin(), it_actual = TypeParam::actual_metric_set.begin(); + it_expected != TypeParam::expected_metric_set.end() && it_actual != TypeParam::actual_metric_set.end(); + it_expected++,it_actual++) { - EXPECT_EQ(itExpected->lane(), itActual->lane()); - EXPECT_EQ(itExpected->tile(), itActual->tile()); - - EXPECT_NEAR(itExpected->clusterDensity(), itActual->clusterDensity(), tol); - EXPECT_NEAR(itExpected->clusterDensityPf(), itActual->clusterDensityPf(), tol); - EXPECT_NEAR(itExpected->clusterCount(), itActual->clusterCount(), tol); - EXPECT_NEAR(itExpected->clusterCountPf(), itActual->clusterCountPf(), tol); - EXPECT_EQ(itExpected->read_metrics().size(), itActual->read_metrics().size()); - for(typename TypeParam::metric_t::read_metric_vector::const_iterator itReadExpected = itExpected->read_metrics().begin(), - itReadActual = itActual->read_metrics().begin(); - itReadExpected != itExpected->read_metrics().end() && - itReadActual != itActual->read_metrics().end(); itReadExpected++, itReadActual++) + EXPECT_EQ(it_expected->lane(), it_actual->lane()); + EXPECT_EQ(it_expected->tile(), it_actual->tile()); + + EXPECT_NEAR(it_expected->cluster_density(), it_actual->cluster_density(), tol); + EXPECT_NEAR(it_expected->cluster_density_pf(), it_actual->cluster_density_pf(), tol); + EXPECT_NEAR(it_expected->cluster_count(), it_actual->cluster_count(), tol); + EXPECT_NEAR(it_expected->cluster_count_pf(), it_actual->cluster_count_pf(), tol); + EXPECT_EQ(it_expected->read_metrics().size(), it_actual->read_metrics().size()); + for(typename TypeParam::metric_t::read_metric_vector::const_iterator it_read_expected = it_expected->read_metrics().begin(), + it_read_actual = it_actual->read_metrics().begin(); + it_read_expected != it_expected->read_metrics().end() && + it_read_actual != it_actual->read_metrics().end(); it_read_expected++, it_read_actual++) { - EXPECT_EQ(itReadExpected->read(), itReadActual->read()); - EXPECT_NEAR(itReadExpected->percent_aligned(), itReadActual->percent_aligned(), tol); - EXPECT_NEAR(itReadExpected->percent_phasing()*scale, itReadActual->percent_phasing(), tol); - EXPECT_NEAR(itReadExpected->percent_prephasing()*scale, itReadActual->percent_prephasing(), tol); + EXPECT_EQ(it_read_expected->read(), it_read_actual->read()); + EXPECT_NEAR(it_read_expected->percent_aligned(), it_read_actual->percent_aligned(), tol); + if(!std::isnan(it_read_expected->percent_phasing()) && !std::isnan(it_read_actual->percent_phasing())) + EXPECT_NEAR(it_read_expected->percent_phasing()*scale, it_read_actual->percent_phasing(), tol); + if(!std::isnan(it_read_expected->percent_prephasing()) && !std::isnan(it_read_actual->percent_prephasing())) + EXPECT_NEAR(it_read_expected->percent_prephasing()*scale, it_read_actual->percent_prephasing(), tol); } } } @@ -190,14 +193,22 @@ TEST(tile_metrics_test, test_tile_metric_for_lane) EXPECT_EQ(expected_metric.tile(), actual_metric.tile()); const float tol = 1e-7f / 0.01f; - EXPECT_NEAR(expected_metric.clusterDensity(), actual_metric.clusterDensity(), tol); - EXPECT_NEAR(expected_metric.clusterDensityPf(), actual_metric.clusterDensityPf(), tol); - EXPECT_NEAR(expected_metric.clusterCount(), actual_metric.clusterCount(), tol); - EXPECT_NEAR(expected_metric.clusterCountPf(), actual_metric.clusterCountPf(), tol); + EXPECT_NEAR(expected_metric.cluster_density(), actual_metric.cluster_density(), tol); + EXPECT_NEAR(expected_metric.cluster_density_pf(), actual_metric.cluster_density_pf(), tol); + EXPECT_NEAR(expected_metric.cluster_count(), actual_metric.cluster_count(), tol); + EXPECT_NEAR(expected_metric.cluster_count_pf(), actual_metric.cluster_count_pf(), tol); EXPECT_EQ(expected_metric.read_metrics().size(), actual_metric.read_metrics().size()); } +TEST(run_metrics_tile_test, test_is_group_empty) +{ + run_metrics metrics; + EXPECT_TRUE(metrics.is_group_empty(constants::Tile)); + std::istringstream fin(tile_v2::binary_data()); + io::read_metrics(fin, metrics.get_set()); + EXPECT_FALSE(metrics.is_group_empty(constants::Tile)); +} #define FIXTURE tile_metrics_test /** diff --git a/src/tests/interop/util/option_parser_test.cpp b/src/tests/interop/util/option_parser_test.cpp index bffde4ddd..e88b72db8 100644 --- a/src/tests/interop/util/option_parser_test.cpp +++ b/src/tests/interop/util/option_parser_test.cpp @@ -10,7 +10,6 @@ #include "interop/util/option_parser.h" #include "interop/model/plot/filter_options.h" - using namespace illumina::interop; diff --git a/tools/build_cov_test.sh b/tools/build_cov_test.sh new file mode 100644 index 000000000..7bdffdf38 --- /dev/null +++ b/tools/build_cov_test.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +######################################################################################################################## +# Test Build Script for Coverity +# +# This script takes three parameters: +# 1. Build Configuration: Debug or Release +# 2. Path to third party binaries: E.g. GTest, NUnit +# 3. Location of Coverity tools +# +# Example running script (from the source directory) +# +# sh tools/build_test.sh Debug /var/external_libs /coverity/bin +# +# Note, you must already have CMake and GCC installed and on your path. And Coverity properly configured for C++. +# +######################################################################################################################## + +# Ensure the script stops on first error +set -e + + +source_dir="../" +build_type="Debug" +build_param="" + +if [ ! -z $1 ] ; then + build_type=$1 +fi +if [ ! -z $2 ] ; then + build_param="-DGTEST_ROOT=$2 -DGMOCK_ROOT=$2 -DNUNIT_ROOT=$2/NUnit-2.6.4" +fi +if [ ! -z $3 ] ; then + cov_path=$3 +fi + +echo "##teamcity[blockOpened name='Configure $build_type']" +mkdir build_cov_${build_type} +cd build_cov_${build_type} +echo "cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param" +cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param +echo "##teamcity[blockClosed name='Configure $build_type']" + +echo "##teamcity[blockOpened name='Coverity Build $build_type']" +$cov_path/cov-build --dir cov make +echo "##teamcity[blockClosed name='Coverity Build $build_type']" + +echo "##teamcity[blockOpened name='Coverity Analyze $build_type']" +$cov_path/cov-analyze --dir cov --strip-path $(dirname `pwd`) --security --concurrency --enable-constraint-fpp --enable-fnptr --enable-virtual +echo "##teamcity[blockClosed name='Coverity Analyze $build_type']" + +echo "##teamcity[blockOpened name='Coverity Upload $build_type']" +$cov_path/cov-commit-defects --dir cov --host ussd-prd-cove01 --stream InterOp_Master --auth-key-file $HOME/key +echo "##teamcity[blockClosed name='Coverity Upload $build_type']" + +cd .. + + diff --git a/tools/build_test.bat b/tools/build_test.bat new file mode 100644 index 000000000..1ef4659bb --- /dev/null +++ b/tools/build_test.bat @@ -0,0 +1,72 @@ +echo off +setlocal enabledelayedexpansion +rem -------------------------------------------------------------------------------------------------------------------- +rem Test Build Script for Windows Compilers +rem +rem The script takes two arguments: +rem 1. Build Configuration: Debug or Release +rem 2. Path to third party binaries: E.g. GTest, NUnit +rem +rem Example Running script (from source directory) +rem .\tools\build_test.bat Debug c:\external +rem +rem Note, you must already have CMake, MinGW and Visual Studio installed and on your path. +rem +rem -------------------------------------------------------------------------------------------------------------------- + +rem -------------------------------------------------------------------------------------------------------------------- +rem MinGW Build Test Script +rem -------------------------------------------------------------------------------------------------------------------- + +set SOURCE_DIR=..\ +set BUILD_PARAM= +set BUILD_TYPE=Debug +if NOT "%1" == "" ( +set BUILD_TYPE=%1 +) +set BUILD_PATH=%2% +if NOT "%2" == "" ( +set BUILD_PARAM=-DGTEST_ROOT=%BUILD_PATH% -DGMOCK_ROOT=%BUILD_PATH% -DNUNIT_ROOT=%BUILD_PATH%/NUnit-2.6.4 +) + +echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% MinGW'] +mkdir build_mingw_%BUILD_TYPE% +cd build_mingw_%BUILD_TYPE% +echo cmake %SOURCE_DIR% -G"MinGW Makefiles" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +cmake %SOURCE_DIR% -G"MinGW Makefiles" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Configure %BUILD_TYPE% MinGW'] + +echo ##teamcity[blockOpened name='Build %BUILD_TYPE% MinGW'] +cmake --build . -- -j8 +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Build %BUILD_TYPE% MinGW'] + +echo ##teamcity[blockOpened name='Test %BUILD_TYPE% MinGW'] +cmake --build . --target check -- -j8 +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Test %BUILD_TYPE% MinGW'] +cd .. + +rem -------------------------------------------------------------------------------------------------------------------- +rem Visual Studio 14 2015 Build Test Script +rem -------------------------------------------------------------------------------------------------------------------- + +echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] +mkdir build_vs2015_x64_%BUILD_TYPE% +cd build_vs2015_x64_%BUILD_TYPE% +echo cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] + +echo ##teamcity[blockOpened name='Build %BUILD_TYPE% Visual Studio 2015 Win64'] +cmake --build . --config %BUILD_TYPE% -- /M +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Build %BUILD_TYPE% Visual Studio 2015 Win64'] + +echo ##teamcity[blockOpened name='Test %BUILD_TYPE% Visual Studio 2015 Win64'] +cmake --build . --target check --config %BUILD_TYPE% -- /M +if %errorlevel% neq 0 exit /b %errorlevel% +echo ##teamcity[blockClosed name='Test %BUILD_TYPE% Visual Studio 2015 Win64'] + diff --git a/tools/build_test.sh b/tools/build_test.sh new file mode 100644 index 000000000..54a796c12 --- /dev/null +++ b/tools/build_test.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +######################################################################################################################## +# Test Build Script for Linux Compilers +# +# This script takes two parameters: +# 1. Build Configuration: Debug or Release +# 2. Path to third party binaries: E.g. GTest, NUnit +# +# Example running script (from the source directory) +# +# sh tools/build_test.sh Debug /var/external_libs +# +# Note, you must already have CMake and GCC installed and on your path. +# +######################################################################################################################## + +# Ensure the script stops on first error +set -e + + +source_dir="../" +build_type="Debug" +build_param="" + +if [ ! -z $1 ] ; then + build_type=$1 +fi +if [ ! -z $2 ] ; then + build_param="-DGTEST_ROOT=$2 -DGMOCK_ROOT=$2 -DNUNIT_ROOT=$2/NUnit-2.6.4" +fi + +echo "##teamcity[blockOpened name='Configure $build_type']" +mkdir build_${build_type} +cd build_${build_type} +echo "cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param" +cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param +echo "##teamcity[blockClosed name='Configure $build_type']" + +echo "##teamcity[blockOpened name='Build $build_type']" +cmake --build . -- -j8 +echo "##teamcity[blockClosed name='Build $build_type']" + +echo "##teamcity[blockOpened name='Test $build_type']" +cmake --build . --target check -- -j8 +echo "##teamcity[blockClosed name='Test $build_type']" +cd .. diff --git a/tools/travis-linux-install.sh b/tools/travis-linux-install.sh index 6cbcb8b14..9ea8017f6 100644 --- a/tools/travis-linux-install.sh +++ b/tools/travis-linux-install.sh @@ -11,5 +11,3 @@ fi - - diff --git a/tools/travis-osx-install.sh b/tools/travis-osx-install.sh index ccaddd504..ac270b92f 100644 --- a/tools/travis-osx-install.sh +++ b/tools/travis-osx-install.sh @@ -10,7 +10,5 @@ brew install swig brew install doxygen brew install https://s3.amazonaws.com/travisbuilds.swig.org/mono.rb -# add swig -# add nunit