From ced299654516ca969437592fdd82ceab5e30ed0c Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Wed, 7 Dec 2016 11:09:18 -0800 Subject: [PATCH 1/8] IPA-5153: Make sure Summary tab calculations are consistent with Confluence documentation --- docs/Doxyfile.in | 2 +- docs/src/binary_formats.md | 8 +- docs/src/calculated_metrics.md | 12 ++ interop/io/format/default_layout.h | 2 +- interop/io/metric_file_stream.h | 1 - interop/io/metric_stream.h | 2 +- interop/io/paths.h | 2 + interop/io/table/csv_format.h | 11 +- interop/logic/plot/plot_qscore_heatmap.h | 1 + interop/logic/utils/channel.h | 20 +- interop/model/metric_base/metric_set.h | 2 +- interop/model/plot/filter_options.h | 1 + interop/model/run/flowcell_layout.h | 2 +- interop/model/run/info.h | 11 + src/apps/inc/application.h | 2 + src/interop/model/metrics/q_metric.cpp | 6 +- src/interop/model/summary/index_summary.cpp | 9 +- src/interop/model/summary/run_summary.cpp | 7 +- .../inc/abstract_regression_test_generator.h | 3 +- src/tests/interop/inc/generic_fixture.h | 6 +- .../interop/inc/proxy_parameter_generator.h | 6 +- .../logic/inc/empty_plot_test_generator.h | 2 +- .../interop/logic/summary_metrics_test.cpp | 38 ++-- .../inc/corrected_intensity_metrics_test.h | 4 +- .../interop/metrics/inc/error_metrics_test.h | 2 +- .../metrics/inc/extraction_metrics_test.h | 2 +- .../interop/metrics/inc/image_metrics_test.h | 4 +- .../interop/metrics/inc/index_metrics_test.h | 2 +- src/tests/interop/metrics/inc/metric_test.h | 1 + .../metrics/inc/q_collapsed_metrics_test.h | 4 +- .../interop/metrics/inc/q_metrics_test.h | 193 +++++++++++++++++- .../interop/metrics/inc/tile_metrics_test.h | 2 +- .../metrics/metric_stream_error_test.cpp | 7 +- 33 files changed, 309 insertions(+), 68 deletions(-) create mode 100644 docs/src/calculated_metrics.md diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index d3e11ba4f..0e72e451c 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -299,7 +299,7 @@ MARKDOWN_SUPPORT = YES # globally by setting AUTOLINK_SUPPORT to NO. # The default value is: YES. -AUTOLINK_SUPPORT = YES +AUTOLINK_SUPPORT = NO # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this diff --git a/docs/src/binary_formats.md b/docs/src/binary_formats.md index f7f339304..ecfbc7b2b 100644 --- a/docs/src/binary_formats.md +++ b/docs/src/binary_formats.md @@ -1,7 +1,7 @@ InterOp Format {#binary_formats} ============== -This pages describes the binary format for an InterOp file. The formats are broken down into two parts: (1) the +This page describes the binary format for an InterOp file. The formats are broken down into two parts: (1) the header, which is written first, a single time for the file and (2) n-Records, which are written next, many times composing the rest of the file. @@ -29,4 +29,8 @@ The documentation for the model notes when an attribute is only populated by a s - @subpage q_collapsed_v5 "Collapsed Q-Metrics Version 5" - @subpage q_collapsed_v6 "Collapsed Q-Metrics Version 6" - + The following are binary formats used only for testing purposes and are not officially supported: + + - @subpage index_summary_v1 "Index Summary v1" + - @subpage summary_v1 "Run Summary v1" + diff --git a/docs/src/calculated_metrics.md b/docs/src/calculated_metrics.md new file mode 100644 index 000000000..ff7eb7638 --- /dev/null +++ b/docs/src/calculated_metrics.md @@ -0,0 +1,12 @@ +# InterOp Calculations {#calculated_metrics} + +This page describes the derived metrics calculated from the InterOp files. + +## Summary Tab + +This section describes each metric shown in the SAV summary tab. + + - @subpage q_metrics_requirement_q30 "% >= Q30" + - @subpage q_metrics_requirement_yield "Yield (G)" + - @subpage q_metrics_requirement_projected_yield "Projected Yield (G)" + diff --git a/interop/io/format/default_layout.h b/interop/io/format/default_layout.h index 36b8387e3..87a96abd5 100644 --- a/interop/io/format/default_layout.h +++ b/interop/io/format/default_layout.h @@ -70,7 +70,7 @@ namespace illumina { namespace interop { namespace io * * This function was originally added to skip control records in tile metrics. * - * @param metric metric to check + * @param id metric to check * @return true, if the metric id is 0 */ template diff --git a/interop/io/metric_file_stream.h b/interop/io/metric_file_stream.h index 8b900a645..c774b1256 100644 --- a/interop/io/metric_file_stream.h +++ b/interop/io/metric_file_stream.h @@ -219,7 +219,6 @@ namespace illumina { namespace interop { namespace io * * @param files destination list of files * @param run_directory file path to the run directory - * @param last_cycle last cycle to check * @param use_out use the copied version */ template diff --git a/interop/io/metric_stream.h b/interop/io/metric_stream.h index 32c049bc3..a0810c1e0 100644 --- a/interop/io/metric_stream.h +++ b/interop/io/metric_stream.h @@ -167,7 +167,7 @@ namespace illumina { namespace interop { namespace io } /** Test whether the format support multi-records * - * @param metric_set header for metric + * @param header header for metric * @param version version of the format * @return true if the format supports multiple records */ diff --git a/interop/io/paths.h b/interop/io/paths.h index f84a34537..c4598b607 100644 --- a/interop/io/paths.h +++ b/interop/io/paths.h @@ -20,6 +20,7 @@ namespace illumina { namespace interop { namespace io public: /** Get the path to the RTA configuration file * + * @param run_directory run directory * @param version version of RTA * @return config filename */ @@ -37,6 +38,7 @@ namespace illumina { namespace interop { namespace io } /** Get the path of the RunParameters * + * @param run_directory run directory * @param alternate return alternate filename * @return path/to/RunParameters.xml */ diff --git a/interop/io/table/csv_format.h b/interop/io/table/csv_format.h index 53953af58..7a4f984d9 100644 --- a/interop/io/table/csv_format.h +++ b/interop/io/table/csv_format.h @@ -35,7 +35,7 @@ namespace illumina { namespace interop { namespace io { namespace table /** Read delimited value from the input stream and cast to proper destination type * * @param in input stream - * @param value string value + * @param buf reusable buffer * @param delim deliminator * @return destintion value */ @@ -49,7 +49,7 @@ namespace illumina { namespace interop { namespace io { namespace table * * @param in input stream * @param dest destination type - * @param value string value + * @param buf reusable buffer * @param delim deliminator */ template @@ -108,9 +108,10 @@ namespace illumina { namespace interop { namespace io { namespace table /** Write a vector of values as a single in a CSV file * * @param out output stream - * @param values source vector of values - * @param beg start column offset - * @param last last column offset + * @param beg iterator to start of collection + * @param end iterator to end of collection + * @param eol end of line terminator character + * @param precision number of digits for floating point number */ template void write_csv(std::ostream& out, I beg, I end, const char eol, const size_t precision=10) diff --git a/interop/logic/plot/plot_qscore_heatmap.h b/interop/logic/plot/plot_qscore_heatmap.h index 9dd80d495..4fd00eb5e 100644 --- a/interop/logic/plot/plot_qscore_heatmap.h +++ b/interop/logic/plot/plot_qscore_heatmap.h @@ -21,6 +21,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot * @param options options to filter the data * @param data output heat map data * @param buffer optional buffer of preallocated memory (for SWIG) + * @param buffer_size number of elements in buffer */ void plot_qscore_heatmap(model::metrics::run_metrics& metrics, const model::plot::filter_options& options, diff --git a/interop/logic/utils/channel.h b/interop/logic/utils/channel.h index 543ffbf3a..c75158355 100644 --- a/interop/logic/utils/channel.h +++ b/interop/logic/utils/channel.h @@ -170,20 +170,26 @@ namespace illumina { namespace interop { namespace logic { namespace utils } /** Update channels from instrument type * + * @param instrument enum type of instrument + * @param channels destination vector for channels */ - inline std::vector update_channel_from_instrument_type(const constants::instrument_type instrument) + inline void update_channel_from_instrument_type(const constants::instrument_type instrument, + std::vector& channels) { - std::vector channels; switch(instrument) { case constants::MiniSeq: case constants::NextSeq: + channels.clear(); + channels.reserve(2); channels.push_back("Red"); channels.push_back("Green"); break; case constants::HiSeq: case constants::MiSeq: case constants::HiScan: + channels.clear(); + channels.reserve(4); channels.push_back("A"); channels.push_back("C"); channels.push_back("G"); @@ -192,6 +198,16 @@ namespace illumina { namespace interop { namespace logic { namespace utils default: break; }; + } + /** Update channels from instrument type + * + * @param instrument enum type of instrument + * @return vector of channel names + */ + inline std::vector update_channel_from_instrument_type(const constants::instrument_type instrument) + { + std::vector channels; + update_channel_from_instrument_type(instrument, channels); return channels; } diff --git a/interop/model/metric_base/metric_set.h b/interop/model/metric_base/metric_set.h index ea88d3a59..0a4beea40 100644 --- a/interop/model/metric_base/metric_set.h +++ b/interop/model/metric_base/metric_set.h @@ -189,7 +189,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base } /** Trim the set to the proper number of metrics * - * @param map proper number of metrics + * @param n actual size of the metric set */ void trim(const size_t n) { diff --git a/interop/model/plot/filter_options.h b/interop/model/plot/filter_options.h index bbfe35978..687dfe50e 100644 --- a/interop/model/plot/filter_options.h +++ b/interop/model/plot/filter_options.h @@ -777,6 +777,7 @@ namespace illumina { namespace interop { namespace model { namespace plot /** Test if metric and plot combination supports filtering by cycle * * @param metric_type metric type + * @param plot_type plot type * @return true if metric supports filtering by cycle */ bool supports_cycle(const constants::metric_type metric_type, const constants::plot_types plot_type)const diff --git a/interop/model/run/flowcell_layout.h b/interop/model/run/flowcell_layout.h index b861f4923..8314495cc 100644 --- a/interop/model/run/flowcell_layout.h +++ b/interop/model/run/flowcell_layout.h @@ -176,7 +176,7 @@ namespace illumina { namespace interop { namespace model { namespace run { m_lane_count = lane_count; } /** Set number of surfaces * - * @param lane_count number of surfaces + * @param surface_count number of surfaces */ void surface_count(const uint_t surface_count) { m_surface_count = surface_count; } diff --git a/interop/model/run/info.h b/interop/model/run/info.h index 4b9652ab2..9c4b2b40b 100644 --- a/interop/model/run/info.h +++ b/interop/model/run/info.h @@ -202,6 +202,17 @@ namespace illumina { namespace interop { namespace model { namespace run if (b->is_index()) return true; return false; } + /** Test if cycle is last cycle of a read + * + * @param cycle cycle number to test + * @return true if the cycle number equals the last cycle of a read + */ + bool is_last_cycle_of_read(const size_t cycle)const + { + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + if (b->last_cycle() == cycle) return true; + return false; + } /** Get read with given number * diff --git a/src/apps/inc/application.h b/src/apps/inc/application.h index 91e58fdd7..4d548b5b2 100644 --- a/src/apps/inc/application.h +++ b/src/apps/inc/application.h @@ -37,6 +37,7 @@ enum exit_codes * * @param filename run folder containing RunInfo.xml and InterOps * @param metrics run metrics + * @param check_empty if true return an error if the metrics are empty * @return exit code */ inline int read_run_metrics(const char* filename, @@ -83,6 +84,7 @@ inline int read_run_metrics(const char* filename, * @param filename run folder containing RunInfo.xml and InterOps * @param metrics run metrics * @param valid_to_load list of metrics that are valid to load + * @param check_empty if true return an error if the metrics are empty * @return exit code */ inline int read_run_metrics(const char* filename, diff --git a/src/interop/model/metrics/q_metric.cpp b/src/interop/model/metrics/q_metric.cpp index ad4e928e7..448f69edb 100644 --- a/src/interop/model/metrics/q_metric.cpp +++ b/src/interop/model/metrics/q_metric.cpp @@ -675,7 +675,7 @@ namespace illumina { namespace interop { namespace io * * This function was originally added to skip control records in tile metrics. * - * @param metric metric to check + * @param id metric to check * @return true, if the metric id is 0 */ template @@ -702,7 +702,7 @@ namespace illumina { namespace interop { namespace io * * This function was originally added to skip control records in tile metrics. * - * @param metric metric to check + * @param id metric to check * @return true, if the metric id is 0 */ template @@ -729,7 +729,7 @@ namespace illumina { namespace interop { namespace io * * This function was originally added to skip control records in tile metrics. * - * @param metric metric to check + * @param id metric to check * @return true, if the metric id is 0 */ template diff --git a/src/interop/model/summary/index_summary.cpp b/src/interop/model/summary/index_summary.cpp index 62501f15d..a49434362 100644 --- a/src/interop/model/summary/index_summary.cpp +++ b/src/interop/model/summary/index_summary.cpp @@ -31,13 +31,10 @@ namespace illumina{ namespace interop{ namespace io { template<> struct generic_layout : public default_layout<1> { - /** @page summary_v1 Index Summary v1 - * - * This class provides an interface to reading the image metric file: - * - InterOp/IndexSummary.bin - * - * The file format for index_flowcell_summary is as follows: + /** @page index_summary_v1 Index Summary v1 * + * This class provides an interface to reading/writing a binary index flowcell summary, and is only supported + * for regression testing. */ /** Map reading/writing a summary to a stream diff --git a/src/interop/model/summary/run_summary.cpp b/src/interop/model/summary/run_summary.cpp index 108f610a5..4eeccace2 100644 --- a/src/interop/model/summary/run_summary.cpp +++ b/src/interop/model/summary/run_summary.cpp @@ -32,11 +32,8 @@ namespace illumina{ namespace interop{ namespace io { { /** @page summary_v1 Run Summary v1 * - * This class provides an interface to reading the image metric file: - * - InterOp/RunSummary.bin - * - * The file format for run_summary is as follows: - * + * This class provides an interface to reading/writing a binary summary, and is only supported + * for regression testing. */ diff --git a/src/tests/interop/inc/abstract_regression_test_generator.h b/src/tests/interop/inc/abstract_regression_test_generator.h index 045f56efe..069ed34f4 100644 --- a/src/tests/interop/inc/abstract_regression_test_generator.h +++ b/src/tests/interop/inc/abstract_regression_test_generator.h @@ -66,7 +66,8 @@ namespace illumina{ namespace interop { namespace unittest * * @param expected expected metric set * @param actual actual metric set - * @return true if the results should be tested (false if rebaselining) + * @param skip_test flag to set if the test should be skipped + * @return success if the results should be tested */ ::testing::AssertionResult generate(Model& expected, Model& actual, bool* skip_test)const { diff --git a/src/tests/interop/inc/generic_fixture.h b/src/tests/interop/inc/generic_fixture.h index 5fe6ccedf..c051db459 100644 --- a/src/tests/interop/inc/generic_fixture.h +++ b/src/tests/interop/inc/generic_fixture.h @@ -38,6 +38,8 @@ namespace illumina{ namespace interop { namespace unittest { * * @param expected expected object * @param actual actual object + * @param skip_test flag to set if the test should be skipped + * @return success if the results should be tested */ virtual ::testing::AssertionResult generate(T& expected, T& actual, bool* skip_test)const=0; /** Advance to the next type @@ -99,7 +101,7 @@ namespace illumina{ namespace interop { namespace unittest { } /** Clone the concrete implementation TODO: Remove * - * @param name run folder + * @param parameter parameter value * @return copy of this object */ base_type operator()(const parameter_type& parameter)const @@ -212,7 +214,7 @@ namespace illumina{ namespace interop { namespace unittest { } /** Copy operator * - * @param other source object to copy + * @param ptr source object to copy * @return this */ generator_ptr& operator=(abstract_generator* ptr) diff --git a/src/tests/interop/inc/proxy_parameter_generator.h b/src/tests/interop/inc/proxy_parameter_generator.h index 3a9322174..5c120a3c6 100644 --- a/src/tests/interop/inc/proxy_parameter_generator.h +++ b/src/tests/interop/inc/proxy_parameter_generator.h @@ -63,8 +63,12 @@ namespace illumina{ namespace interop { namespace unittest * @param obj object-like functor * @param base generator base * @param it iterator to std::vector + * @param it_end iterator to end of std::vector */ - proxy_argument_iterator(T& obj, const proxy_argument_generator& base, const_iterator it, const_iterator it_end) : + proxy_argument_iterator(T& obj, + const proxy_argument_generator& base, + const_iterator it, + const_iterator it_end) : m_base(base), m_begin(it), m_current(it), m_end(it_end), m_object(obj) { static_assert(!is_pointer::value, "This class does not free memory and should not take a pointer"); diff --git a/src/tests/interop/logic/inc/empty_plot_test_generator.h b/src/tests/interop/logic/inc/empty_plot_test_generator.h index e57b5cd10..e664b8514 100644 --- a/src/tests/interop/logic/inc/empty_plot_test_generator.h +++ b/src/tests/interop/logic/inc/empty_plot_test_generator.h @@ -39,7 +39,7 @@ namespace illumina { namespace interop { namespace unittest } /** Clone the concrete implementation TODO: Remove * - * @param name run folder + * @param plot_type type of plot * @return copy of this object */ base_type operator()(const constants::plot_types plot_type)const diff --git a/src/tests/interop/logic/summary_metrics_test.cpp b/src/tests/interop/logic/summary_metrics_test.cpp index 99268e48a..9fa8ee417 100644 --- a/src/tests/interop/logic/summary_metrics_test.cpp +++ b/src/tests/interop/logic/summary_metrics_test.cpp @@ -11,6 +11,7 @@ #include #include "interop/util/math.h" #include "interop/logic/summary/run_summary.h" +#include "interop/logic/utils/channel.h" #include "src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h" #include "src/tests/interop/metrics/inc/error_metrics_test.h" #include "src/tests/interop/metrics/inc/extraction_metrics_test.h" @@ -91,6 +92,7 @@ ::testing::AssertionResult AreCycleStatesNear(const model::run::cycle_range& exp } if(expected.last_cycle() != actual.last_cycle()) { + if(test_failed) msg << " | "; msg << "Last Cycle Expected: " << expected.last_cycle() << " == Actual: " << actual.last_cycle(); test_failed = true; } @@ -349,8 +351,8 @@ TEST(summary_metrics_test, cycle_35_cycle_34_tile) const size_t surface_count = 2; const size_t swath_count = 4; const size_t tile_count = 99; - const size_t sections_per_lane = 6; - const size_t lanes_per_section = 6; + const size_t sections_per_lane = 1; + const size_t lanes_per_section = 1; std::vector channels; channels.push_back("Red"); channels.push_back("Green"); @@ -415,8 +417,8 @@ TEST(summary_metrics_test, clear_run_metrics) // TODO Expand to catch everything const size_t surface_count = 2; const size_t swath_count = 4; const size_t tile_count = 99; - const size_t sections_per_lane = 6; - const size_t lanes_per_section = 6; + const size_t sections_per_lane = 1; + const size_t lanes_per_section = 1; std::vector channels; channels.push_back("Red"); channels.push_back("Green"); @@ -439,7 +441,7 @@ TEST(summary_metrics_test, clear_run_metrics) // TODO Expand to catch everything run_info.set_naming_method(constants::FourDigit); model::metrics::run_metrics full_metrics(run_info); - tile_metric_v2::create_expected(full_metrics.get()); + tile_metric_v2::create_expected(full_metrics.get(), run_info); model::summary::run_summary summary; logic::summary::summarize_run_metrics(full_metrics, summary); INTEROP_EXPECT_NEAR(summary.total_summary().percent_aligned(), 2.5863409042358398f, tol); @@ -466,8 +468,6 @@ TEST(summary_metrics_test, empty_run_metrics) EXPECT_EQ(summary.size(), 0u); } - - //--------------------------------------------------------------------------------------------------------------------- // Unit test section //--------------------------------------------------------------------------------------------------------------------- @@ -515,20 +515,21 @@ class run_summary_generator : public abstract_generator(expected.surface_count()); std::vector reads; expected.copy_reads(reads); std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); + logic::utils::update_channel_from_instrument_type( + expected.channel_count() == 2 ? constants::NextSeq : constants::HiSeq, channels ); actual = model::summary::run_summary(reads.begin(), reads.end(), lane_count, surface_count, channels.size()); model::run::info run_info("XX", "", @@ -542,9 +543,9 @@ class run_summary_generator : public abstract_generator()); + Gen::create_expected(metrics.get(), run_info); metrics.finalize_after_load(); SummaryLogic summary_logic; summary_logic(metrics, actual); @@ -619,6 +620,9 @@ run_summary_tests::generator_type run_summary_unit_test_generators[] = { new run_summary_generator(), new run_summary_generator(), + // Requirements testing + new run_summary_generator(), + // Write/read wrap(new standard_parameter_generator(0)) }; diff --git a/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h b/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h index 99ac8040e..52d90dc25 100644 --- a/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h +++ b/src/tests/interop/metrics/inc/corrected_intensity_metrics_test.h @@ -25,7 +25,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { typedef metric_t::uint_t uint_t; typedef metric_t::ushort_t ushort_t; @@ -95,7 +95,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { typedef metric_t::uint_t uint_t; typedef metric_t::ushort_t ushort_t; diff --git a/src/tests/interop/metrics/inc/error_metrics_test.h b/src/tests/interop/metrics/inc/error_metrics_test.h index ac7471ee8..a49ee11f8 100644 --- a/src/tests/interop/metrics/inc/error_metrics_test.h +++ b/src/tests/interop/metrics/inc/error_metrics_test.h @@ -28,7 +28,7 @@ namespace illumina { namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t &metrics) + static void create_expected(metric_set_t &metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(VERSION); metrics.insert(metric_t(7, 1114, 1, 0.450100899f)); diff --git a/src/tests/interop/metrics/inc/extraction_metrics_test.h b/src/tests/interop/metrics/inc/extraction_metrics_test.h index 24da4c652..4061be89f 100644 --- a/src/tests/interop/metrics/inc/extraction_metrics_test.h +++ b/src/tests/interop/metrics/inc/extraction_metrics_test.h @@ -27,7 +27,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(VERSION); diff --git a/src/tests/interop/metrics/inc/image_metrics_test.h b/src/tests/interop/metrics/inc/image_metrics_test.h index 2ceb1ed72..7ad0f5470 100644 --- a/src/tests/interop/metrics/inc/image_metrics_test.h +++ b/src/tests/interop/metrics/inc/image_metrics_test.h @@ -30,7 +30,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(VERSION); typedef metric_t::ushort_t ushort_t; @@ -83,7 +83,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(header_t(2), VERSION); typedef metric_t::ushort_t ushort_t; diff --git a/src/tests/interop/metrics/inc/index_metrics_test.h b/src/tests/interop/metrics/inc/index_metrics_test.h index 9366c452c..3241eaf33 100644 --- a/src/tests/interop/metrics/inc/index_metrics_test.h +++ b/src/tests/interop/metrics/inc/index_metrics_test.h @@ -34,7 +34,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { typedef metric_t::index_info_t index_info_t; metrics = metric_set_t(VERSION); diff --git a/src/tests/interop/metrics/inc/metric_test.h b/src/tests/interop/metrics/inc/metric_test.h index a715581fe..67b04647b 100644 --- a/src/tests/interop/metrics/inc/metric_test.h +++ b/src/tests/interop/metrics/inc/metric_test.h @@ -10,6 +10,7 @@ #include #include #include "interop/model/metric_base/metric_set.h" +#include "interop/model/run/info.h" #include "interop/io/metric_file_stream.h" #include "interop/util/length_of.h" diff --git a/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h b/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h index d1e87ec09..052b9e12c 100644 --- a/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h +++ b/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h @@ -32,7 +32,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(parent_t::VERSION); metrics.insert(metric_t(1,1105,1,2447414,2334829,2566750,33)); @@ -100,7 +100,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { typedef typename metric_set_t::header_type header_t; typedef typename header_t::qscore_bin_vector_type qscore_bin_vector_type; diff --git a/src/tests/interop/metrics/inc/q_metrics_test.h b/src/tests/interop/metrics/inc/q_metrics_test.h index 65f35e405..bd59a1f48 100644 --- a/src/tests/interop/metrics/inc/q_metrics_test.h +++ b/src/tests/interop/metrics/inc/q_metrics_test.h @@ -27,7 +27,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(VERSION); @@ -114,7 +114,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { typedef header_t::qscore_bin_vector_type qscore_bin_vector_type; typedef header_t::bin_t bin_t; @@ -214,7 +214,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { typedef header_t::qscore_bin_vector_type qscore_bin_vector_type; typedef header_t::bin_t bin_t; @@ -300,7 +300,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(VERSION); typedef metric_t::uint_t uint_t; @@ -349,5 +349,190 @@ namespace illumina{ namespace interop { namespace unittest } }; + /** + * @page q_metrics_requirement_q30 % >= Q30 + * + * The QMetricsOut.bin InterOp file contains a histogram of the counts of PF clusters at each quality value ranging + * from 1 to 50 for each lane, tile, and cycle. % >= Q30 is calculated as the sum of the populations in bins with + * a quality value of 30 or greater divided by the total non-N basecalls (sum of the population over all bins times + * 100). Note that we perform these calculations on usable cycles only (in other words, we exclude the last cycle + * of each read). + * + * @sa illumina::interop::unittest::q_metric_requirements + * + * @section q_metrics_requirement_q30_lane Lane + * + * For all tiles over all lanes for a given read, for all usable cycles (i.e. not counting the last + * cycle of the read), the sum of the populations in bins with a quality value of 30 or greater divided + * by the sum of the population over all bins times 100. + * + * @section q_metrics_requirement_q30_read Read + * + * For all tiles in a lane for a given read, for all usable cycles (i.e. not counting the last + * cycle of the read), the sum of the populations in bins with a quality value of 30 or greater + * divided by the sum of the population over all bins times 100. + * + * @section q_metrics_requirement_q30_non_index Non-Indexed Total + * + * For all tiles over all non-indexed reads, for all usable cycles (i.e. not counting the last cycle of + * each read), the sum of the populations in bins with a quality value of 30 or greater divided by the sum + * of the population over all bins times 100. + * + * @section q_metrics_requirement_q30_total Total + * + * For all tiles over all reads, for all usable cycles (i.e. not counting the last cycle of each read), the + * sum of the populations in bins with a quality value of 30 or greater divided by the sum of the + * population over all bins times 100. + * + * + * @page q_metrics_requirement_yield Yield (G) + * + * @sa illumina::interop::unittest::q_metric_requirements + * + * @section q_metrics_requirement_yield_lane Lane + * + * The sum of bases from passing filter clusters that are not N over all cycles, except the last cycle, + * divided by 1E9 to put in units of gigabases. + * + * @section q_metrics_requirement_yield_read Read + * + * The sum of bases from passing filter clusters that are not N over all cycles, except the last cycle, + * divided by 1E9 to put in units of gigabases. + * + * @section q_metrics_requirement_yield_non_index Non-Indexed Total + * + * The sum of the read yields over all non-indexed reads. + * + * @section q_metrics_requirement_yield_total Total + * + * The sum of the read yields over all reads. + * + * + * @page q_metrics_requirement_projected_yield Projected Yield (G) + * + * @sa illumina::interop::unittest::q_metric_requirements + * + * @section q_metrics_requirement_projected_yield_read Read + * + * The sum of bases from passing filter clusters that are not N over all cycles, except the last cycle, + * scaled to the total read length, divided by 1E9 to put in units of gigabases. The scaling factor is + * defined, per lane, as the final number of entries (tiles in the lane x usable cycles) divided by the + * current number of entries for the read/lane in the QMetrics file. + * + * @section q_metrics_requirement_projected_yield_non_index Non-Indexed Total + * + * The sum of the projected total yield over all reads, excluding indexed reads. + * + * @section q_metrics_requirement_projected_yield_total Total + * + * The sum of the projected total yield over all reads, including indexed reads. + */ + + /** This generator creates a contrived run metrics and q-metrics to test specific requirements + * + * + * @see model::metrics::q_metric + * @note Version 6 + * + */ + struct q_metric_requirements : metric_test + { + /** Create the expected metric set + * + * @param metrics destination metric set + * @param run_info run info + */ + static void create_expected(metric_set_t& metrics, const model::run::info& run_info) + { + typedef model::metrics::q_metric::uint_t uint_t; + model::metrics::q_score_bin bins[] = + { + model::metrics::q_score_bin(2,9,2), + model::metrics::q_score_bin(10,19,14), + model::metrics::q_score_bin(20,24,21), + model::metrics::q_score_bin(25,29,27), + model::metrics::q_score_bin(30,34,32), + model::metrics::q_score_bin(35,39,36), + model::metrics::q_score_bin(40,40,40) + }; + metrics = model::metrics::q_metric::header_type(util::to_vector(bins)); + + const ::uint32_t kValue1 = static_cast< ::uint32_t>(1e7); + const ::uint32_t kValue2 = static_cast< ::uint32_t>(2e7); + // >= Q30 -> 50.0 + const ::uint32_t hist[] = {kValue2, kValue2, kValue1, 0, kValue1, kValue2, kValue2}; + // >= Q30 -> 42.85 // 3/7 + const ::uint32_t hist_last_cycle_of_read[] = {kValue2, kValue2, kValue2, kValue1, kValue1, kValue1, kValue1}; + for(size_t cycle=1;cycle<=run_info.total_cycles();++cycle) + { + if(run_info.is_last_cycle_of_read(cycle)) + { + metrics.insert( + model::metrics::q_metric(1, 1101, static_cast(cycle), util::to_vector(hist_last_cycle_of_read))); + metrics.insert( model::metrics::q_metric(1, 1102, static_cast(cycle), util::to_vector(hist_last_cycle_of_read))); + } + else + { + metrics.insert( model::metrics::q_metric(1, 1101, static_cast(cycle), util::to_vector(hist))); + metrics.insert( model::metrics::q_metric(1, 1102, static_cast(cycle), util::to_vector(hist))); + } + } + } + /** Create the expected run_summary to test the Q-metrics requirements + * + * @param summary run summary + */ + static void create_summary(model::summary::run_summary& summary) + { + const size_t tile_count = 2; + const size_t lane_count = 1; + const size_t lane_number = 1; + const size_t surface_count = 2; + const size_t channel_count = 2; + const size_t cycles_per_read = 6; + const float kExpectedPercentGreaterThanOrEqualToQ30 = 50.0f;// 7 bins were half the data >= Q30 + const float kExpectedReadYield = 1.0f; // 1e9 / 1e9 + + + const model::run::read_info reads[]={ + model::run::read_info(1, 1, cycles_per_read), + model::run::read_info(2, cycles_per_read+1, cycles_per_read*2) + }; + const size_t read_count = util::length_of(reads); + const size_t total_cycles = cycles_per_read*read_count; + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); + + for(size_t read_index=0;read_index < read_count;++read_index) + { + summary[read_index][0].lane(lane_number); + summary[read_index][0].tile_count(tile_count); + summary[read_index][0][0].tile_count(tile_count); + // TODO: Describe? + summary[read_index][0].cycle_state().qscored_cycle_range( + model::run::cycle_range(cycles_per_read, cycles_per_read)); + summary[read_index].summary().percent_gt_q30(kExpectedPercentGreaterThanOrEqualToQ30); + summary[read_index][0].percent_gt_q30(kExpectedPercentGreaterThanOrEqualToQ30); + summary[read_index][0][0].percent_gt_q30(kExpectedPercentGreaterThanOrEqualToQ30); + summary[read_index][0].yield_g(kExpectedReadYield); + summary[read_index][0][0].yield_g(kExpectedReadYield); + summary[read_index].summary().yield_g(kExpectedReadYield); + summary[read_index].summary().projected_yield_g(kExpectedReadYield); + + // TODO: Describe? + summary[read_index][0].projected_yield_g(kExpectedReadYield); + summary[read_index][0][0].projected_yield_g(kExpectedReadYield); + } + summary.nonindex_summary().percent_gt_q30(kExpectedPercentGreaterThanOrEqualToQ30); + summary.total_summary().percent_gt_q30(kExpectedPercentGreaterThanOrEqualToQ30); + summary.nonindex_summary().yield_g(kExpectedReadYield*read_count); + summary.total_summary().yield_g(kExpectedReadYield*read_count); + summary.total_summary().projected_yield_g(kExpectedReadYield*read_count); + + summary.nonindex_summary().projected_yield_g(kExpectedReadYield*read_count); + // TODO: Describe? + summary.cycle_state().qscored_cycle_range(model::run::cycle_range(total_cycles, total_cycles)); + } + }; + }}} diff --git a/src/tests/interop/metrics/inc/tile_metrics_test.h b/src/tests/interop/metrics/inc/tile_metrics_test.h index c8eb27370..c6b545098 100644 --- a/src/tests/interop/metrics/inc/tile_metrics_test.h +++ b/src/tests/interop/metrics/inc/tile_metrics_test.h @@ -33,7 +33,7 @@ namespace illumina{ namespace interop { namespace unittest * * @param metrics destination metric set */ - static void create_expected(metric_set_t& metrics) + static void create_expected(metric_set_t& metrics, const model::run::info=model::run::info()) { metrics = metric_set_t(VERSION); diff --git a/src/tests/interop/metrics/metric_stream_error_test.cpp b/src/tests/interop/metrics/metric_stream_error_test.cpp index 07bbd2111..c35330d08 100644 --- a/src/tests/interop/metrics/metric_stream_error_test.cpp +++ b/src/tests/interop/metrics/metric_stream_error_test.cpp @@ -115,10 +115,11 @@ TYPED_TEST_P(metric_stream_error_test, test_hardcoded_read) TEST(metric_stream_error_test, image_metric_out_of_bounds) { typedef model::metrics::image_metric::ushort_t ushort_t; - model::metric_base::metric_set metric_set( - model::metrics::image_metric_header(2)); const ushort_t min_vals[] = {100, 200, 300}; - metric_set.insert( model::metrics::image_metric(1, 1101, 1, 2, util::to_vector(min_vals), util::to_vector(min_vals)) ); + model::metric_base::metric_set metric_set( // Force an out of bounds condition + model::metrics::image_metric_header(static_cast(util::length_of(min_vals)-1))); + metric_set.insert( + model::metrics::image_metric(1, 1101, 1, 2, util::to_vector(min_vals), util::to_vector(min_vals))); std::ostringstream fout; EXPECT_THROW(io::write_metrics(fout, metric_set), io::bad_format_exception); } From 1fca67d2372d5714da9d435f60ee4b39f848a446 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Tue, 13 Dec 2016 13:29:04 -0800 Subject: [PATCH 2/8] Clean up unused enum types --- interop/model/metric_base/base_cycle_metric.h | 13 ++-- interop/model/metric_base/base_metric.h | 59 ++++++++++++++----- interop/model/metric_base/base_read_metric.h | 13 ++-- interop/model/metric_base/metric_set.h | 11 ++-- src/ext/swig/exceptions/exceptions_csharp.i | 2 + .../logic/table/create_imaging_table.cpp | 3 +- 6 files changed, 68 insertions(+), 33 deletions(-) diff --git a/interop/model/metric_base/base_cycle_metric.h b/interop/model/metric_base/base_cycle_metric.h index aa658290c..efb850cca 100644 --- a/interop/model/metric_base/base_cycle_metric.h +++ b/interop/model/metric_base/base_cycle_metric.h @@ -72,12 +72,6 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ class base_cycle_metric : public base_metric { - public: - enum - { - /** Base for records written out once for each tile/cycle */ - BASE_TYPE = constants::BaseCycleType - }; public: /** A cycle metric_set header */ @@ -189,6 +183,13 @@ namespace illumina { namespace interop { namespace model { namespace metric_base private: uint_t m_cycle; }; + + + + /** Specialization for base_cycle_metric providing sentinel values*/ + template<> + struct metric_attributes : public metric_attributes{}; + }}}} diff --git a/interop/model/metric_base/base_metric.h b/interop/model/metric_base/base_metric.h index 7e00bd018..b4fe1a533 100644 --- a/interop/model/metric_base/base_metric.h +++ b/interop/model/metric_base/base_metric.h @@ -26,6 +26,10 @@ namespace illumina { namespace interop { namespace model { namespace metric_base // Forward declaration class base_metric; + /** Get attributes of the metric */ + template + struct metric_attributes : public Metric{}; + /** Defines default base header for metric */ class base_metric_header : public empty_header { @@ -92,7 +96,6 @@ namespace illumina { namespace interop { namespace model { namespace metric_base typedef constants::base_tile_t base_t; enum { - TYPE=constants::UnknownMetricType, //Compress Lane, Tile and cycle into a 64-bit Integer // Bit order: LTCR or Lane, Tile, Cycle/Read, Reserved // Reserved: Bits 0-16 @@ -106,13 +109,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base RESERVED_BIT_COUNT = 16, // Support up to 65535 values READ_BIT_SHIFT = RESERVED_BIT_COUNT, CYCLE_BIT_SHIFT = RESERVED_BIT_COUNT, - EVENT_BIT_SHIFT = 0, TILE_BIT_SHIFT = CYCLE_BIT_SHIFT+CYCLE_BIT_COUNT, - LANE_BIT_SHIFT = TILE_BIT_SHIFT+TILE_BIT_COUNT, // Supports up to 63 lanes - /** Base for records written out once for each tile */ - BASE_TYPE = constants::BaseTileType, - /** Latest version is sentinel*/ - LATEST_VERSION=0 + LANE_BIT_SHIFT = TILE_BIT_SHIFT+TILE_BIT_COUNT // Supports up to 63 lanes= }; public: /** Constructor @@ -135,12 +133,6 @@ namespace illumina { namespace interop { namespace model { namespace metric_base m_lane = lane; m_tile = tile; } - /** Get the metric name suffix - * - * @return empty string - */ - static const char *suffix() - { return ""; } /** Set the base metric identifiers * @@ -424,12 +416,14 @@ namespace illumina { namespace interop { namespace model { namespace metric_base return m_tile - 1; } } - - /** Empty prefix string + /** Get the metric name suffix * * @return empty string */ - static const char *prefix() { return ""; } + static const char *suffix() + { + return ""; + } private: uint_t m_lane; @@ -437,5 +431,38 @@ namespace illumina { namespace interop { namespace model { namespace metric_base }; + /** Specialization for base_metric providing sentinel values*/ + template<> + struct metric_attributes + { + enum + { + /** Metric type identifier + * @sa illumina::interop::constants::metric_group + */ + TYPE = constants::UnknownMetricType, + /** Latest version of the metric format*/ + LATEST_VERSION=0 + }; + + /** Empty prefix string + * + * @return empty string + */ + static const char *prefix() + { + return ""; + } + /** Get the metric name suffix + * + * @return empty string + */ + static const char *suffix() + { + return ""; + } + }; + + }}}} diff --git a/interop/model/metric_base/base_read_metric.h b/interop/model/metric_base/base_read_metric.h index f2fd10fd0..5a6fa730a 100644 --- a/interop/model/metric_base/base_read_metric.h +++ b/interop/model/metric_base/base_read_metric.h @@ -57,12 +57,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * * These classes define both a lane, tile and read identifier. */ - class base_read_metric : public base_metric { + class base_read_metric : public base_metric + { public: - enum { - /** Base for records written out once for each tile/read */ - BASE_TYPE = constants::BaseReadType - }; /** A read metric_set header */ typedef base_read_metric_header header_type; @@ -153,6 +150,12 @@ namespace illumina { namespace interop { namespace model { namespace metric_base private: uint_t m_read; }; + + /** Specialization for base_cycle_metric providing sentinel values*/ + template<> + struct metric_attributes : public metric_attributes{}; + + }}}} diff --git a/interop/model/metric_base/metric_set.h b/interop/model/metric_base/metric_set.h index ea88d3a59..7b5687b73 100644 --- a/interop/model/metric_base/metric_set.h +++ b/interop/model/metric_base/metric_set.h @@ -71,11 +71,12 @@ namespace illumina { namespace interop { namespace model { namespace metric_base typedef typename metric_array_t::const_iterator const_iterator; /** Metric iterator */ typedef typename metric_array_t::iterator iterator; - enum { + enum + { /** Group type enum */ - TYPE=T::TYPE, + TYPE=metric_attributes::TYPE, /** Latest version of the format */ - LATEST_VERSION=T::LATEST_VERSION + LATEST_VERSION=metric_attributes::LATEST_VERSION }; public: @@ -551,14 +552,14 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @return prefix */ static const char *prefix() - { return T::prefix(); } + { return metric_attributes::prefix(); } /** Get the suffix of the InterOp filename * * @return suffix */ static const char *suffix() - { return T::suffix(); } + { return metric_attributes::suffix(); } public: /** Get metric for lane, tile and cycle diff --git a/src/ext/swig/exceptions/exceptions_csharp.i b/src/ext/swig/exceptions/exceptions_csharp.i index d0118783e..31ad21921 100644 --- a/src/ext/swig/exceptions/exceptions_csharp.i +++ b/src/ext/swig/exceptions/exceptions_csharp.i @@ -40,7 +40,9 @@ ENTRY_POINT(customDelegate); } } + #pragma warning disable 0414 // Need to keep a reference to this object static CustomExceptionHelper_##EXCEPTION_CPLUS_PLUS exceptionHelper_##EXCEPTION_CPLUS_PLUS = new CustomExceptionHelper_##EXCEPTION_CPLUS_PLUS(); + #pragma warning restore 0414 %} diff --git a/src/interop/logic/table/create_imaging_table.cpp b/src/interop/logic/table/create_imaging_table.cpp index f083fa086..63d5a146b 100644 --- a/src/interop/logic/table/create_imaging_table.cpp +++ b/src/interop/logic/table/create_imaging_table.cpp @@ -323,10 +323,11 @@ namespace illumina { namespace interop { namespace logic { namespace table using namespace constants; using namespace model::table; using namespace model; + using namespace model::metric_base; // TODO: This can be reduced to a single macro define typedef std::pair mapped_t; # define INTEROP_TUPLE7(Id, Metric, Ignore2, Ignore3, Ignore4, Ignore5, Ignored6) \ - mapped_t(Id##Column,static_cast(Metric::TYPE)), + mapped_t(Id##Column,static_cast(metric_attributes::TYPE)), static const mapped_t name_types[] = {INTEROP_IMAGING_COLUMN_TYPES mapped_t(ImagingColumnCount, UnknownMetricGroup)}; # undef INTEROP_TUPLE7 return util::constant_mapping_get(name_types, type, UnknownMetricGroup); From 67b4fb8c68332d69b329183d697abc949a652524 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Wed, 14 Dec 2016 10:50:42 -0800 Subject: [PATCH 3/8] Enhance C# version information --- cmake/AssemblyInfo.cs.in | 4 ++-- cmake/package.nuspec.in | 2 ++ cmake/version.rc.in | 2 +- src/ext/csharp/CMakeLists.txt | 1 + src/interop/logic/table/create_imaging_table.cpp | 9 +++++---- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmake/AssemblyInfo.cs.in b/cmake/AssemblyInfo.cs.in index b9048db86..a30df49b6 100644 --- a/cmake/AssemblyInfo.cs.in +++ b/cmake/AssemblyInfo.cs.in @@ -1,8 +1,8 @@ using System.Reflection; -[assembly: AssemblyTitle("Illumina.InterOp")] +[assembly: AssemblyTitle("Illumina.InterOp with @CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@")] [assembly: AssemblyCompany("Illumina")] -[assembly: AssemblyProduct("Illumina.InterOp")] +[assembly: AssemblyProduct("Illumina.InterOp - @VERSION@ ")] [assembly: AssemblyCopyright("Copyright © Illumina 2015")] [assembly: AssemblyVersion("@VERSION_SHORT@")] [assembly: AssemblyFileVersion("@VERSION_SHORT@")] diff --git a/cmake/package.nuspec.in b/cmake/package.nuspec.in index 7408fcfa7..d45d37b3a 100644 --- a/cmake/package.nuspec.in +++ b/cmake/package.nuspec.in @@ -14,6 +14,8 @@ The Illumina InterOp libraries are a set of common routines used for reading InterOp metric files produced by Illumina sequencers. These libraries are backwards compatible and capable of supporting prior releases of the software, with one exception: GA systems have been excluded. + + C++ Compiler: @COMPILER_INFO@ High-level C++ binding for Illumina C++ InterOp Library, packaged for diff --git a/cmake/version.rc.in b/cmake/version.rc.in index 52da2fb89..bb204d6e0 100644 --- a/cmake/version.rc.in +++ b/cmake/version.rc.in @@ -31,7 +31,7 @@ BEGIN BEGIN BLOCK "040904b0" BEGIN - VALUE "FileDescription", "Illumina InterOp Library\0" + VALUE "FileDescription", "Illumina InterOp with @CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@\0" VALUE "ProductVersion", "@VERSION@\0" VALUE "FileVersion", "@VERSION@\0" VALUE "InternalName", "@LIB_NAME@\0" diff --git a/src/ext/csharp/CMakeLists.txt b/src/ext/csharp/CMakeLists.txt index 6b4ddc925..1ff51b0dd 100644 --- a/src/ext/csharp/CMakeLists.txt +++ b/src/ext/csharp/CMakeLists.txt @@ -147,6 +147,7 @@ add_custom_target(nuspec ${CMAKE_SOURCE_DIR}/cmake/package.targets $/../illumina_interop_${CSHARP_TYPE}_${NUGET_SYS_ID}.targets COMMAND ${CMAKE_COMMAND} + -DCOMPILER_INFO="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" -DCMAKE_CURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR} -DCSHARP_TYPE=${CSHARP_TYPE} -DCSHARP_VERSION=${CSHARP_VERSION} diff --git a/src/interop/logic/table/create_imaging_table.cpp b/src/interop/logic/table/create_imaging_table.cpp index f083fa086..3c6ee907d 100644 --- a/src/interop/logic/table/create_imaging_table.cpp +++ b/src/interop/logic/table/create_imaging_table.cpp @@ -50,7 +50,7 @@ namespace illumina { namespace interop { namespace logic { namespace table { const id_t id = beg->cycle_hash(); typename row_offset_map_t::const_iterator row_it = row_offset.find(id); - const size_t row = row_it->second; + const ::uint64_t row = row_it->second; const summary::read_cycle& read = cycle_to_read[beg->cycle()-1]; if(data_beg[row*column_count]==0) { @@ -83,7 +83,8 @@ namespace illumina { namespace interop { namespace logic { namespace table 0, naming_method, columns, - data_beg+row*column_count, data_end); + data_beg+row*column_count, + data_end); } } /** Populate the imaging table with a by cycle InterOp metric set @@ -217,7 +218,7 @@ namespace illumina { namespace interop { namespace logic { namespace table const id_t tid = model::metric_base::base_cycle_metric::tile_hash_from_id(it->first); if (!tile_metrics.has_metric(tid)) continue; const id_t cycle = model::metric_base::base_cycle_metric::cycle_from_id(it->first); - const size_t row = it->second; + const ::uint64_t row = it->second; INTEROP_ASSERTMSG(cycle <= cycle_to_read.size(), cycle << " <= " << cycle_to_read.size() << " tile: " << model::metric_base::base_cycle_metric::tile_from_id(it->first)); @@ -277,7 +278,7 @@ namespace illumina { namespace interop { namespace logic { namespace table cycle_metric_map_t hash_set; metrics.populate_id_map(hash_set); row_offset.clear(); - size_t row = 0; + ::uint64_t row = 0; for(cycle_metric_map_t::const_iterator it = hash_set.begin();it != hash_set.end();++it,++row) row_offset[it->first]=row; } From b9fbd468e241f5f5184630599b720a4c8b097378 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Thu, 15 Dec 2016 13:57:33 -0800 Subject: [PATCH 4/8] IPA-5885: Ensure error summary matches requirements --- appveyor.yml | 2 +- docs/src/calculated_metrics.md | 1 + interop/logic/summary/error_summary.h | 78 +++++-- interop/model/run/info.h | 11 + src/ext/csharp/CMakeLists.txt | 2 +- src/ext/swig/metrics.i | 10 +- src/ext/swig/plot.i | 2 + src/ext/swig/run.i | 10 +- src/ext/swig/util/operator_overload.i | 5 +- src/tests/interop/inc/generic_fixture.h | 22 +- .../interop/logic/summary_metrics_test.cpp | 42 ++-- .../interop/metrics/inc/error_metrics_test.h | 198 ++++++++++++++++++ 12 files changed, 326 insertions(+), 57 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d3f041aa8..b3de6209f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,7 +42,7 @@ environment: # config_type: win32 - generator: "Visual Studio 12 2013 Win64" BUILD: msvc12_win64 - config_type: win64 + config_type: win64_not_supported # - generator: "Visual Studio 12 2013" # BUILD: msvc12_win32 # config_type: win32 diff --git a/docs/src/calculated_metrics.md b/docs/src/calculated_metrics.md index ff7eb7638..29c6a495d 100644 --- a/docs/src/calculated_metrics.md +++ b/docs/src/calculated_metrics.md @@ -9,4 +9,5 @@ This section describes each metric shown in the SAV summary tab. - @subpage q_metrics_requirement_q30 "% >= Q30" - @subpage q_metrics_requirement_yield "Yield (G)" - @subpage q_metrics_requirement_projected_yield "Projected Yield (G)" + - @subpage error_metrics_requirement "% Error" diff --git a/interop/logic/summary/error_summary.h b/interop/logic/summary/error_summary.h index dacdb948c..137ae8f99 100644 --- a/interop/logic/summary/error_summary.h +++ b/interop/logic/summary/error_summary.h @@ -22,6 +22,62 @@ namespace illumina { namespace interop { namespace logic { namespace summary { + /** Cache data structure for error metrics + * + */ + class error_cache_element + { + public: + /** Constructor */ + error_cache_element() : m_error_sum(0.0f), m_error_count(0), m_max_cycle(0) {} + /** Update the cache element + * + * @param error_rate error rate + */ + void update_error(const float error_rate) + { + if(std::isnan(error_rate)) return; + m_error_sum += error_rate; + m_error_count++; + } + /** Update the cache element + * + * @param cycle_within_read cycle within read + */ + void update_cycle(const size_t cycle_within_read) + { + m_max_cycle = std::max(m_max_cycle, cycle_within_read); + } + /** Average error rate + * + * @return average + */ + float average()const + { + return divide(m_error_sum, static_cast(m_error_count)); + } + /** Get the maximum cycle apart of the average + * + * @return maximum cycle + */ + size_t max_cycle()const + { + return m_max_cycle; + } + /** Test if there are no error values + * + * @return true if no values have been averaged + */ + bool is_empty()const + { + return m_error_count == 0; + } + + private: + float m_error_sum; + size_t m_error_count; + size_t m_max_cycle; + }; /** Cache errors for all tiles up to a give max cycle * @@ -45,13 +101,10 @@ namespace illumina { namespace interop { namespace logic { namespace summary summary_by_lane_read &read_lane_surface_cache) throw(model::index_out_of_bounds_exception) { - typedef std::vector cycle_vector_t; typedef std::pair key_t; - typedef std::pair value_t; - typedef INTEROP_ORDERED_MAP(key_t, value_t) error_tile_t; + typedef INTEROP_ORDERED_MAP(key_t, error_cache_element) error_tile_t; typedef std::vector error_by_read_tile_t; error_by_read_tile_t tmp(read_lane_cache.size()); - cycle_vector_t max_error_cycle(read_lane_cache.size(), 0); for (; beg != end; ++beg) { INTEROP_ASSERT(beg->cycle() > 0); @@ -59,19 +112,13 @@ namespace illumina { namespace interop { namespace logic { namespace summary if ((beg->cycle() - 1) >= cycle_to_read.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Cycle exceeds total cycles from Reads in the RunInfo.xml"); const read_cycle &read = cycle_to_read[beg->cycle() - 1]; - if (read.cycle_within_read > max_cycle || read.is_last_cycle_in_read) continue; const key_t key = std::make_pair(beg->lane(), beg->tile()); const size_t read_number = read.number - 1; - max_error_cycle[read_number] = std::max(max_error_cycle[read_number], - static_cast(read.cycle_within_read)); INTEROP_ASSERTMSG(read_number < tmp.size(), read.number << " " << read.cycle_within_read << ", " << beg->cycle()); - if (tmp[read_number].find(key) == tmp[read_number].end()) - { - tmp[read_number].insert(std::make_pair(key, std::make_pair(0.0f, 0.0f))); - } - tmp[read_number][key].first += beg->error_rate(); - tmp[read_number][key].second += 1; + tmp[read_number][key].update_cycle(read.cycle_within_read); + if (read.cycle_within_read > max_cycle || read.is_last_cycle_in_read) continue; + tmp[read_number][key].update_error(beg->error_rate()); } for (size_t read = 0; read < tmp.size(); ++read) { @@ -82,8 +129,9 @@ namespace illumina { namespace interop { namespace logic { namespace summary const size_t lane = ebeg->first.first - 1; if (lane >= read_lane_cache.lane_count()) INTEROP_THROW(model::index_out_of_bounds_exception, "Lane exceeds number of lanes in RunInfo.xml"); - if(max_cycle < std::numeric_limits::max() && ebeg->second.second < max_cycle) continue; - const float err_avg = divide(ebeg->second.first, ebeg->second.second); + if(max_cycle < std::numeric_limits::max() && ebeg->second.max_cycle() < max_cycle) continue; + if(ebeg->second.is_empty()) continue; + const float err_avg = ebeg->second.average(); read_lane_cache(read, lane).push_back(err_avg); if(read_lane_surface_cache.surface_count() < 2) continue; const ::uint32_t surface = logic::metric::surface(static_cast< ::uint32_t >(ebeg->first.second), naming_method); diff --git a/interop/model/run/info.h b/interop/model/run/info.h index 9c4b2b40b..a5e98913a 100644 --- a/interop/model/run/info.h +++ b/interop/model/run/info.h @@ -213,6 +213,17 @@ namespace illumina { namespace interop { namespace model { namespace run if (b->last_cycle() == cycle) return true; return false; } + /** Get the cycle number within a read (0 is returned for invalid cycles) + * + * @param cycle cycle number to test + * @return cycle number within read + */ + size_t cycle_within_read(const size_t cycle)const + { + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + if ( cycle <= b->last_cycle()) return cycle-b->first_cycle() + 1; + return 0; + } /** Get read with given number * diff --git a/src/ext/csharp/CMakeLists.txt b/src/ext/csharp/CMakeLists.txt index 1ff51b0dd..1d4d3b036 100644 --- a/src/ext/csharp/CMakeLists.txt +++ b/src/ext/csharp/CMakeLists.txt @@ -70,7 +70,7 @@ set(SWIG_GEN_CSHARP_SOURCE_FILES ${CMAKE_SWIG_OUTDIR}/*.cs CACHE INTERNAL "C# so set(ASSEMBLY_INFO ${CMAKE_CURRENT_BINARY_DIR}/AssemblyInfo.cs) configure_file(${CMAKE_SOURCE_DIR}/cmake/AssemblyInfo.cs.in ${ASSEMBLY_INFO} @ONLY) -csharp_add_library(csharp_interop ${CMAKE_SWIG_OUTDIR}/*.cs ${ASSEMBLY_INFO}) +csharp_add_library(csharp_interop ${CMAKE_SWIG_OUTDIR}/*.cs) add_dependencies(csharp_interop ${SWIG_MODULE_LIST}) foreach(SRC ${SWIG_SRCS}) get_filename_component(MODULE ${SRC} NAME_WE) diff --git a/src/ext/swig/metrics.i b/src/ext/swig/metrics.i index 67e82d9a5..c3a32c532 100644 --- a/src/ext/swig/metrics.i +++ b/src/ext/swig/metrics.i @@ -1,4 +1,4 @@ -/** Metrics model and logic +/** Metrics model and metric logic */ %include %include @@ -36,12 +36,6 @@ using Illumina.InterOp.Run; EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Ignore methods -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Wrap the base metrics //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -161,6 +155,8 @@ WRAP_METRICS(WRAP_TYPES) WRAP_METRICS(WRAP_METRIC_SET) + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Metric Logic //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/ext/swig/plot.i b/src/ext/swig/plot.i index d1aa13f0a..b2b8e8e00 100644 --- a/src/ext/swig/plot.i +++ b/src/ext/swig/plot.i @@ -43,6 +43,8 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Plotting //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +%ignore illumina::interop::model::plot::flowcell_data::tile_id(size_t const,size_t const); %{ #include "interop/model/plot/axes.h" #include "interop/model/plot/candle_stick_point.h" diff --git a/src/ext/swig/run.i b/src/ext/swig/run.i index 4f9499652..5f6052de4 100644 --- a/src/ext/swig/run.i +++ b/src/ext/swig/run.i @@ -7,7 +7,12 @@ %include "src/ext/swig/exceptions/exceptions_impl.i" %include "util/operator_overload.i" -%rename(to_int) operator uint64_t; +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Ignore methods that should not be wrapped +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +%ignore illumina::interop::model::metrics::q_score_bin::operator=; +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + %pragma(java) jniclasscode=%{ static { @@ -15,7 +20,6 @@ } %} -%ignore operator enum_t; %{ #include "interop/interop.h" @@ -29,6 +33,8 @@ #include "interop/logic/utils/enums.h" %} + + // Primitive array types %template(string_vector) std::vector< std::string >; %template(ulong_vector) std::vector< uint64_t >; diff --git a/src/ext/swig/util/operator_overload.i b/src/ext/swig/util/operator_overload.i index 4878e4583..e43098129 100644 --- a/src/ext/swig/util/operator_overload.i +++ b/src/ext/swig/util/operator_overload.i @@ -1,15 +1,15 @@ // Suppress warnings and expose operators to client languages // Most languages do no support operator overloading like C++, so these need to be renamed or ignored. -#if defined(SWIGCSHARP) || defined(SWIGJAVA) +#if defined(SWIGCSHARP) || defined(SWIGJAVA) || defined(SWIGPYTHON) // TODO: Add python section and rename to appropriate python operator %rename(equals) operator==; - %rename(subtract) operator-; %rename(add) *::operator+=; %rename(is_less) operator< const; %rename(copy) operator=; + %rename(to_int) operator uint64_t; #else // Do nothing #endif @@ -20,6 +20,7 @@ // TODO: Remove at in C++ and rename operator[] to at %ignore *::operator[]; +%ignore operator enum_t; diff --git a/src/tests/interop/inc/generic_fixture.h b/src/tests/interop/inc/generic_fixture.h index c051db459..63f4e7ae6 100644 --- a/src/tests/interop/inc/generic_fixture.h +++ b/src/tests/interop/inc/generic_fixture.h @@ -347,6 +347,20 @@ namespace illumina{ namespace interop { namespace unittest { const ::testing::AssertionResult fixture_test_result; }; + /** Test if two floats are nearly the same If both are NaN, then this check succeeds + * + * @param expected expected float + * @param actual actual float + * @param tol tolerance + * @return rue if both numbers hold the same value, or their difference is less than tolerance + */ + inline bool is_float_near(const float expected, const float actual, const float tol) + { + if(std::isnan(expected) && std::isnan(actual)) return true; + if(std::isnan(expected) || std::isnan(actual)) return false; + return std::abs(expected-actual) < tol; + } + /** Check if two floats are nearly the same. If both are NaN, then this check succeeds * * @todo Use this everywhere @@ -358,10 +372,7 @@ namespace illumina{ namespace interop { namespace unittest { */ inline ::testing::AssertionResult AreFloatsNear(const float expected, const float actual, const float tol) { - if(std::isnan(expected) && std::isnan(actual)) return ::testing::AssertionSuccess(); - if(std::isnan(expected) || std::isnan(actual)) - return ::testing::AssertionFailure() << "Abs(" << expected << " - " << actual << ") >= " << tol; - if(std::abs(expected-actual) < tol) return ::testing::AssertionSuccess(); + if(is_float_near(expected, actual, tol)) return ::testing::AssertionSuccess(); return ::testing::AssertionFailure() << "Abs(" << expected << " - " << actual << ") >= " << tol; } @@ -386,8 +397,7 @@ namespace illumina{ namespace interop { namespace unittest { } for(size_t i=0;i= tol) + if(!is_float_near(expected[i], actual[i], tol)) { if(test_failed) msg << " | "; msg << "Value("<< i << ") Expected: " << expected[i] << " == Actual: " << actual[i]; diff --git a/src/tests/interop/logic/summary_metrics_test.cpp b/src/tests/interop/logic/summary_metrics_test.cpp index 9fa8ee417..9d29efb43 100644 --- a/src/tests/interop/logic/summary_metrics_test.cpp +++ b/src/tests/interop/logic/summary_metrics_test.cpp @@ -42,31 +42,22 @@ ::testing::AssertionResult AreStatsNear(const metric_stat& expected, const metri { ::testing::Message msg; bool test_failed = false; - if(!std::isnan(expected.mean()) || !std::isnan(actual.mean())) + if(!is_float_near(expected.mean(), actual.mean(), tol)) { - if(std::abs(expected.mean()-actual.mean()) >= tol) - { - msg << "Mean Expected: " << expected.mean() << " == Actual: " << actual.mean(); - test_failed=true; - } + msg << "Mean Expected: " << expected.mean() << " == Actual: " << actual.mean(); + test_failed=true; } - if(!std::isnan(expected.stddev()) || !std::isnan(actual.stddev())) + if(!is_float_near(expected.stddev(), actual.stddev(), tol)) { - if(std::abs(expected.stddev()-actual.stddev()) >= tol) - { - if(test_failed) msg << " | "; - msg << "StdDev Expected: " << expected.stddev() << " == Actual: " << actual.stddev(); - test_failed=true; - } + if(test_failed) msg << " | "; + msg << "StdDev Expected: " << expected.stddev() << " == Actual: " << actual.stddev(); + test_failed=true; } - if(!std::isnan(expected.median()) || !std::isnan(actual.median())) + if(!is_float_near(expected.median(), actual.median(), tol)) { - if(std::abs(expected.median()-actual.median()) >= tol) - { - if(test_failed) msg << " | "; - msg << "Median Expected: " << expected.median() << " == Actual: " << actual.median(); - test_failed=true; - } + if(test_failed) msg << " | "; + msg << "Median Expected: " << expected.median() << " == Actual: " << actual.median(); + test_failed=true; } if(test_failed) return ::testing::AssertionFailure(msg << " Tol: " << tol); return ::testing::AssertionSuccess(); @@ -161,7 +152,9 @@ TEST_P(run_summary_tests, read_summary) SCOPED_TRACE(trace_message); EXPECT_EQ(actual_read_summary.size(), expected_read_summary.size()); EXPECT_EQ(actual_read_summary.lane_count(), expected_read_summary.lane_count()); - INTEROP_EXPECT_NEAR(actual_read_summary.summary().error_rate(), expected_read_summary.summary().error_rate(), tol); + INTEROP_EXPECT_NEAR(actual_read_summary.summary().error_rate(), + expected_read_summary.summary().error_rate(), + tol); INTEROP_EXPECT_NEAR(actual_read_summary.summary().percent_aligned(), expected_read_summary.summary().percent_aligned(), tol); INTEROP_EXPECT_NEAR(actual_read_summary.summary().first_cycle_intensity(), @@ -201,10 +194,12 @@ TEST_P(run_summary_tests, lane_summary) for (size_t lane = 0; lane < expected[read].size(); ++lane) { ::testing::Message trace_message; - trace_message << "Read Index: " << read << " - Lane Index: " << lane; - SCOPED_TRACE(trace_message); const model::summary::lane_summary &actual_lane_summary = actual[read][lane]; const model::summary::lane_summary &expected_lane_summary = expected[read][lane]; + trace_message << "Read Index: " << read + << " - Lane Index: " << lane + << " - Lane Number: " << expected_lane_summary.lane(); + SCOPED_TRACE(trace_message); EXPECT_EQ(actual_lane_summary.lane(), expected_lane_summary.lane()); EXPECT_GT(actual_lane_summary.lane(), 0u); EXPECT_EQ(actual_lane_summary.tile_count(), @@ -622,6 +617,7 @@ run_summary_tests::generator_type run_summary_unit_test_generators[] = { // Requirements testing new run_summary_generator(), + new run_summary_generator(), // Write/read wrap(new standard_parameter_generator(0)) diff --git a/src/tests/interop/metrics/inc/error_metrics_test.h b/src/tests/interop/metrics/inc/error_metrics_test.h index a49ee11f8..7485ad3b2 100644 --- a/src/tests/interop/metrics/inc/error_metrics_test.h +++ b/src/tests/interop/metrics/inc/error_metrics_test.h @@ -83,6 +83,204 @@ namespace illumina { namespace interop { namespace unittest } }; + /** + * @page error_metrics_requirement Error Metrics + * + * The ErrorMetricsOut.bin InterOp file contains a the error in Phi X alignment for each tile on each cycle. + * + * @sa illumina::interop::unittest::error_metric_requirements + * + * @section error_metrics_requirement_35_lane % Error After 35 Cycles Per Lane + * + * The average (+/- the standard deviation) of the Phi X error rate for all tiles in a lane for the first 35 + * cycles of a given read. If an error rate value is not present for any of the first 35 cycles in the read for + * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a + * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, + * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these + * statistics. + * + * @section error_metrics_requirement_50_lane % Error After 50 Cycles Per Lane + * + * The average (+/- the standard deviation) of the Phi X error rate for all tiles in a lane for the first 50 + * cycles of a given read. If an error rate value is not present for any of the first 50 cycles in the read for + * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a + * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, + * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these + * statistics. + * + * @section error_metrics_requirement_75_lane % Error After 75 Cycles Per Lane + * + * The average (+/- the standard deviation) of the Phi X error rate for all tiles in a lane for the first 75 + * cycles of a given read. If an error rate value is not present for any of the first 75 cycles in the read for + * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a + * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, + * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these + * statistics. + * + * @section error_metrics_requirement_100_lane % Error After 100 Cycles Per Lane + * + * The average (+/- the standard deviation) of the Phi X error rate for all tiles in a lane for the first 100 + * cycles of a given read. If an error rate value is not present for any of the first 100 cycles in the read for + * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a + * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, + * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these + * statistics. + * + * @section error_metrics_requirement_error % Error + * + * @subsection error_metrics_requirement_error_lane Lane + * + * The average (+/- the standard deviation) of the Phi X error rate for all tiles in a lane for a given read. If + * an error rate value is not present for any cycles in the read for a given tile, it is excluded from the + * averaging. If no tiles have any error statistics for a given read, a value of 0 is returned. Note that the + * base unit for this is the error rate for a single tile - in other words, we average the per-cycle error + * rate across all the relevant cycles for a tile before calculating these statistics + * + * @subsection error_metrics_requirement_error_read Read + * + * The average of the Phi X error rate for all tiles over all lanes for a given read. If an error rate value is + * not present for any cycles in the read for a given tile, it is excluded from the averaging. If no tiles have + * any error statistics for a given read, a value of 0 is returned. Note that the base unit for this is the error + * rate for a single tile - in other words, we average the per-cycle error rate across all the relevant cycles for + * a tile before calculating these statistics + * + * @subsection error_metrics_requirement_error_non_index Non-Indexed Total + * + * The average of the Phi X error rate for all tiles over all non-indexed reads. If an error rate value is not + * present for any cycles in the read for a given tile, it is excluded from the averaging. If no tiles have any + * error statistics for any non-indexed read, a value of 0 is returned. Note that the base unit for this is the + * error rate for a single tile - in other words, we average the per-cycle error rate across all the relevant + * cycles for a tile before calculating these statistics + * + * @subsection error_metrics_requirement_error_total Total + * + * The average of the Phi X error rate for all tiles over all reads. If an error rate value is not present for + * any cycles in the read for a given tile, it is excluded from the averaging. If no tiles have any error + * statistics for any read, a value of 0 is returned. Note that the base unit for this is the error rate for + * a single tile - in other words, we average the per-cycle error rate across all the relevant cycles for a + * tile before calculating these statistics + * + */ + + /** This generator creates a contrived run info and error-metrics to test specific requirements + * + * @see model::metrics::error_metric + * @note Version 3 + */ + struct error_metric_requirements : metric_test + { + /** Create the expected metric set + * + * @param metrics destination metric set + * @param run_info run info + */ + static void create_expected(metric_set_t &metrics, const model::run::info &run_info) + { + metrics = metric_set_t(VERSION); + + const float kMissingValue = std::numeric_limits::quiet_NaN(); + const float kAverageError35 = 1.0f; + const float kAverageError50 = 4.0f; + const float kAverageError75 = 16.0f; + const float kAverageError100 = 26.0f; + const float kAverageErrorAfter = 36.0f; + + float sum = 0.0f; + const size_t cycle_count = run_info.total_cycles(); + for (size_t cycle = 1; cycle <= cycle_count; ++cycle) + { + const size_t cycle_within_read = run_info.cycle_within_read(cycle); + const metric_t::uint_t cycle_id = static_cast(cycle); + metrics.insert(metric_t(7, 1113, cycle_id, kMissingValue)); + if( cycle_within_read <= 5 ) + { + //metrics.insert(metric_t(7, 1114, cycle, kMissingValue)); + } + else if( cycle_within_read <= 35 ) + { + metrics.insert(metric_t(7, 1114, cycle_id, kAverageError35)); + sum += kAverageError35; + } + else if( cycle_within_read <= 50 ) + { + metrics.insert(metric_t(7, 1114, cycle_id, kAverageError50)); + sum += kAverageError35; + } + else if( cycle_within_read <= 75 ) + { + metrics.insert(metric_t(7, 1114, cycle_id, kAverageError75)); + } + else if( cycle_within_read <= 100 ) + { + metrics.insert(metric_t(7, 1114, cycle_id, kAverageError100)); + } + else + { + metrics.insert(metric_t(7, 1114, cycle_id, kAverageErrorAfter)); + } + } + } + + /** Create the expected run_summary to test the error metrics requirements + * + * @param summary run summary + */ + static void create_summary(model::summary::run_summary &summary) + { + const size_t lane_count = 1; + const size_t surface_count = 2; + const size_t channel_count = 2; + const size_t cycles_per_read = 126; + + const model::run::read_info reads[]={ + model::run::read_info(1, 1, cycles_per_read), + model::run::read_info(2, cycles_per_read+1, cycles_per_read*2) + }; + summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); + + const size_t read_count = util::length_of(reads); + const size_t total_cycles = cycles_per_read*read_count; + const size_t tile_count = 2; + const size_t lane_number = 7; + const float kReadError35 = 1.0f; + const float kReadError50 = 2.0f; + const float kReadError75 = 7.0f; + const float kReadError100 = 12.0f; + const float kReadError = 17.0f; + const float kSigma = 0.0f; + + for(size_t read_index=0;read_index < read_count;++read_index) + { + summary[read_index][0].lane(lane_number); + summary[read_index][0].tile_count(tile_count); + summary[read_index][0][0].tile_count(tile_count); + summary[read_index][0].cycle_state().error_cycle_range( + model::run::cycle_range(cycles_per_read, cycles_per_read)); + + // Lane Error rate + summary[read_index][0].error_rate_35(model::summary::metric_stat(kReadError35,kSigma,kReadError35)); + summary[read_index][0][0].error_rate_35(model::summary::metric_stat(kReadError35,kSigma,kReadError35)); + + summary[read_index][0].error_rate_50(model::summary::metric_stat(kReadError50,kSigma,kReadError50)); + summary[read_index][0][0].error_rate_50(model::summary::metric_stat(kReadError50,kSigma,kReadError50)); + + summary[read_index][0].error_rate_75(model::summary::metric_stat(kReadError75,kSigma,kReadError75)); + summary[read_index][0][0].error_rate_75(model::summary::metric_stat(kReadError75,kSigma,kReadError75)); + + summary[read_index][0].error_rate_100(model::summary::metric_stat(kReadError100,kSigma,kReadError100)); + summary[read_index][0][0].error_rate_100(model::summary::metric_stat(kReadError100,kSigma,kReadError100)); + + summary[read_index][0].error_rate(model::summary::metric_stat(kReadError,kSigma,kReadError)); + summary[read_index][0][0].error_rate(model::summary::metric_stat(kReadError,kSigma,kReadError)); + + // Read Error Rate + summary[read_index].summary().error_rate(kReadError); + } + summary.nonindex_summary().error_rate(kReadError); + summary.total_summary().error_rate(kReadError); + summary.cycle_state().error_cycle_range(model::run::cycle_range(total_cycles, total_cycles)); + } + }; }}} From 21fc4113adc27b0bf21d065493fc4a13eede5dff Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Mon, 19 Dec 2016 13:36:59 -0800 Subject: [PATCH 5/8] Removes the coverity run from the build matrix for every pull request --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96d5d90d1..d29c5d5d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,6 @@ language: cpp sudo: required -env: - global: - - secure: "oALD7yjwurT4h4x/iATz8zP+q9y1bOG8l9wFucmM8HTIdThN+37YZDmLnhlRWKDZNt6NwBRom1vHmTZAmLGO0RohPxRV8RrceQGq5Jjq0gmXQafxjhbZetKxFbXKJ5/4RdUM0yxPCmYCscHEJG/1gk+F6y7hFiRPO8A5z8AhNabCZ8BMwaLaWpko7mHO1HqEs/MWExj32IF0DVQtzWyJeyO4RYkYKtBGfsGZMHfhsis3Nhfql5shN64AmlAxJewxApJg0wbnf5fbXyLYqZS92MW0GbPph+grdpEGEgGVgyGRV0Ys9EAj0CQs5ukmi+JP6TL2BbBnmreAOwiWMQDGzAKgwipOH3Phmcor4bFo5TU0vTBhP1RCxnPa/zyTrwI94KH96b+j2WT/6wDkXrixhVAeJntuWEhbJvklN/lC8/nlGnbRfyEB5Ists5yFzERxqIWd3x+qtknJO8/wcetbavBFUPcgEZdTXVv1zTquzLeUNlavseT9egFFQ7cDnGwg8b9LWb+sQeqiTjT3p6WQ/4+tw/EVn+xCfqkw746Gy8aXFdviJU3mcYeR7gvgYO84HXaoS0Mv7mxmJeWfoNopkv0U7cPNS3eXU3XZ2bg0WZl90i2LLXcQgtm8nz3NVEhGfpgjlQ+WBQcVOYG93vBcOa4powoCHojgzZn55T/J3nQ=" - matrix: include: - compiler: clang @@ -36,24 +32,6 @@ matrix: - compiler: gcc os: linux env: INTEROP_C89=ON BUILD_TYPE=Release BUILD_NAME=linux_gcc46_release DEPLOY_BUILD=true - - compiler: gcc - os: linux - env: INTEROP_C89=ON BUILD_TYPE=Release BUILD_NAME=linux_gcc46_release COVERITY=1 - addons: - apt: - sources: - - george-edison55-precise-backports - packages: - - cmake - - cmake-data - coverity_scan: - project: - name: "Illumina/interop" - description: "InterOp Libraries for Illumina Sequencers" - build_command_prepend: "cmake .. -DENABLE_SWIG=OFF -DENABLE_DOCS=OFF" - build_command: "cmake --build ." - notification_email: ezralanglois+coverity@gmail.com - branch_pattern: master - compiler: gcc addons: &1 apt: @@ -141,7 +119,6 @@ before_script: - export CXX="${CXX}${COMPILER_VERSION}" script: -- if test ! -z ${COVERITY}; then exit 0; fi - export BUILD_PATH=${PWD}/${BUILD_NAME} - echo 'Configuring...' && echo -en 'travis_fold:start:script.1\\r' - cmake -DNUNIT_ROOT=$HOME/NUnit-2.6.4 ../ -DENABLE_BACKWARDS_COMPATIBILITY=$INTEROP_C89 -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=${BUILD_PATH} From acd0d1595928320f8bb3012d031a54db211ff04f Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Tue, 20 Dec 2016 12:51:46 -0800 Subject: [PATCH 6/8] IPA-5923: Fix bug in legacy q-metric binning --- interop/logic/metric/q_metric.h | 10 +++ interop/model/metrics/q_metric.h | 13 ++++ interop/model/run_metrics.h | 20 +++++- src/interop/logic/metric/q_metric.cpp | 34 ++++++++++ src/interop/model/run_metrics.cpp | 60 +++++++++++++---- .../interop/metrics/inc/q_metrics_test.h | 65 ++++++++++++------- 6 files changed, 162 insertions(+), 40 deletions(-) diff --git a/interop/logic/metric/q_metric.h b/interop/logic/metric/q_metric.h index 4f66931f7..743c3a3d6 100644 --- a/interop/logic/metric/q_metric.h +++ b/interop/logic/metric/q_metric.h @@ -105,6 +105,16 @@ namespace illumina { namespace interop { namespace logic { namespace metric void populate_legacy_q_score_bins(std::vector& q_score_bins, const constants::instrument_type instrument, const size_t count); + /** Compress the q-metric set using the bins in the header + * + * @param q_metric_set q-metric set + */ + void compress_q_metrics(model::metric_base::metric_set& q_metric_set); + /** Compress the q-metric set using the bins in the header + * + * @param q_metric_set q-metric set + */ + void compress_q_metrics(model::metric_base::metric_set& q_metric_set); /** Populate the q-score header bins from the data * * This only for legacy platforms that use older q-metric formats, which do not include bin information diff --git a/interop/model/metrics/q_metric.h b/interop/model/metrics/q_metric.h index a9ae7b355..cde388be9 100644 --- a/interop/model/metrics/q_metric.h +++ b/interop/model/metrics/q_metric.h @@ -583,6 +583,19 @@ namespace illumina { namespace interop { namespace model { namespace metrics (*it) += (*cur); } } + /** Compress bins + * + * @param header binned header + */ + void compress(const header_type& header) + { + if(size() == header.bin_count() || header.bin_count() == 0) return; + for(size_t i=0;i(header.bin_at(i).value()-1)]; + } + m_qscore_hist.resize(header.bin_count()); + } /** Q-score value of the histogram * diff --git a/interop/model/run_metrics.h b/interop/model/run_metrics.h index 0402aa3ce..7f4047fa1 100644 --- a/interop/model/run_metrics.h +++ b/interop/model/run_metrics.h @@ -113,7 +113,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics io::file_not_found_exception, io::bad_format_exception, io::incomplete_file_exception, - io::format_exception, + model::invalid_channel_exception, model::index_out_of_bounds_exception, model::invalid_tile_naming_method, model::invalid_run_info_exception); @@ -131,7 +131,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics io::file_not_found_exception, io::bad_format_exception, io::incomplete_file_exception, - io::format_exception, + model::invalid_channel_exception, model::index_out_of_bounds_exception, model::invalid_tile_naming_method, model::invalid_run_info_exception, @@ -173,7 +173,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @param count number of bins for legacy q-metrics */ - void finalize_after_load(size_t count = std::numeric_limits::max()) throw(io::format_exception, + void finalize_after_load(size_t count = std::numeric_limits::max()) throw(model::invalid_channel_exception, model::invalid_tile_naming_method, model::index_out_of_bounds_exception, model::invalid_run_info_exception); @@ -195,6 +195,20 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param naming_method tile naming method */ void set_naming_method(const constants::tile_naming_method naming_method); + /** Get number of legacy bins + * + * @param legacy_bin_count known number of bins + * @return number of legacy bins + */ + size_t count_legacy_bins(const size_t legacy_bin_count=std::numeric_limits::max())const; + /** Test whether run parameters must be loaded + * + * This is used to determine channel count and legacy q-score bins + * + * @param legacy_bin_count known number of bins + * @return true if run parameters is required + */ + bool is_run_parameters_required(const size_t legacy_bin_count=std::numeric_limits::max())const; public: /** Get information about the run diff --git a/src/interop/logic/metric/q_metric.cpp b/src/interop/logic/metric/q_metric.cpp index f0ea7e994..b97013d09 100644 --- a/src/interop/logic/metric/q_metric.cpp +++ b/src/interop/logic/metric/q_metric.cpp @@ -147,6 +147,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric const size_t count) { typedef model::metrics::q_score_bin q_score_bin; + if(!q_score_bins.empty())return; if(!requires_legacy_bins(count)) return; q_score_bins.reserve(count); if(instrument == constants::NextSeq) @@ -265,5 +266,38 @@ namespace illumina { namespace interop { namespace logic { namespace metric } } + /** Compress the q-metric set using the bins in the header + * + * @param q_metric_set q-metric set + */ + template + void compress_q_metrics_t(model::metric_base::metric_set& q_metric_set) + { + typedef model::metric_base::metric_set metric_set_t; + if(q_metric_set.empty()) return; + if(logic::metric::is_compressed(q_metric_set) || q_metric_set.bin_count() == 0 ) return; + for(typename metric_set_t::iterator it = q_metric_set.begin();it != q_metric_set.end();++it) + { + it->compress(q_metric_set); + } + } + + /** Compress the q-metric set using the bins in the header + * + * @param q_metric_set q-metric set + */ + void compress_q_metrics(model::metric_base::metric_set& q_metric_set) + { + compress_q_metrics_t(q_metric_set); + } + /** Compress the q-metric set using the bins in the header + * + * @param q_metric_set q-metric set + */ + void compress_q_metrics(model::metric_base::metric_set& q_metric_set) + { + compress_q_metrics_t(q_metric_set); + } + }}}} diff --git a/src/interop/model/run_metrics.cpp b/src/interop/model/run_metrics.cpp index 57dad8772..d8f88e304 100644 --- a/src/interop/model/run_metrics.cpp +++ b/src/interop/model/run_metrics.cpp @@ -317,7 +317,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics io::file_not_found_exception, io::bad_format_exception, io::incomplete_file_exception, - io::format_exception, + model::invalid_channel_exception, model::index_out_of_bounds_exception, model::invalid_tile_naming_method, model::invalid_run_info_exception) @@ -341,15 +341,16 @@ namespace illumina { namespace interop { namespace model { namespace metrics io::file_not_found_exception, io::bad_format_exception, io::incomplete_file_exception, - io::format_exception, + model::invalid_channel_exception, model::index_out_of_bounds_exception, model::invalid_tile_naming_method, model::invalid_run_info_exception, invalid_parameter) { clear(); - const size_t count = read_xml(run_folder); + read_run_info(run_folder); read_metrics(run_folder, valid_to_load); + const size_t count = read_run_parameters(run_folder); finalize_after_load(count); check_for_data_sources(run_folder); } @@ -395,10 +396,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics xml::missing_xml_element_exception, xml::xml_parse_exception) { - const size_t count = logic::metric::count_legacy_q_score_bins(get()); + const size_t count = count_legacy_bins(); if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count)) { - try { m_run_parameters.read(run_folder); @@ -415,13 +415,42 @@ namespace illumina { namespace interop { namespace model { namespace metrics } return count; } + /** Test whether run parameters must be loaded + * + * This is used to determine channel count and legacy q-score bins + * + * @param legacy_bin_count known number of bins + * @return true if run parameters is required + */ + bool run_metrics::is_run_parameters_required(const size_t legacy_bin_count)const + { + return m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count_legacy_bins(legacy_bin_count)); + } + /** Get number of legacy bins + * + * @param legacy_bin_count known number of bins + * @return number of legacy bins + */ + size_t run_metrics::count_legacy_bins(const size_t legacy_bin_count)const + { + if( legacy_bin_count < std::numeric_limits::max() ) return legacy_bin_count; + if( !get().empty() ) + { + return logic::metric::count_legacy_q_score_bins(get()); + } + else if( !get().empty() ) + { + return logic::metric::count_legacy_q_score_bins(get()); + } + return std::numeric_limits::max(); + } /** Finalize the metric sets after loading from disk * * @param count number of bins for legacy q-metrics */ void run_metrics::finalize_after_load(size_t count) - throw(io::format_exception, + throw(model::invalid_channel_exception, model::invalid_tile_naming_method, model::index_out_of_bounds_exception, model::invalid_run_info_exception) @@ -434,13 +463,18 @@ namespace illumina { namespace interop { namespace model { namespace metrics } if (count == std::numeric_limits::max()) { - if (get().size() > 0) - count = logic::metric::count_legacy_q_score_bins(get()); - else if (get().size()) - count = logic::metric::count_legacy_q_score_bins(get()); + count = count_legacy_bins(); + } + if(logic::metric::requires_legacy_bins(count)) + { + logic::metric::populate_legacy_q_score_bins(get().bins(), m_run_parameters.instrument_type(), + count); + logic::metric::populate_legacy_q_score_bins(get().bins(), + m_run_parameters.instrument_type(), + count); + logic::metric::compress_q_metrics(get()); + logic::metric::compress_q_metrics(get()); } - logic::metric::populate_legacy_q_score_bins(get().bins(), m_run_parameters.instrument_type(), - count); if (get().size() > 0 && get().size() == 0) { logic::metric::create_collapse_q_metrics(get(), get()); @@ -459,7 +493,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics { legacy_channel_update(m_run_parameters.instrument_type()); if (m_run_info.channels().empty()) - INTEROP_THROW(io::format_exception, + INTEROP_THROW(model::invalid_channel_exception, "Channel names are missing from the RunInfo.xml, and RunParameters.xml does not contain sufficient information on the instrument run."); } if (!empty()) diff --git a/src/tests/interop/metrics/inc/q_metrics_test.h b/src/tests/interop/metrics/inc/q_metrics_test.h index bd59a1f48..b1fde80f8 100644 --- a/src/tests/interop/metrics/inc/q_metrics_test.h +++ b/src/tests/interop/metrics/inc/q_metrics_test.h @@ -33,10 +33,9 @@ namespace illumina{ namespace interop { namespace unittest typedef metric_t::uint_t uint_t; typedef sparse_value q_val; - - const q_val hist1[] = {q_val(14,21208), q_val(21,8227), q_val(26,73051), q_val(32,2339486)}; - const q_val hist2[] = {q_val(14,22647), q_val(21,9570), q_val(26,81839), q_val(32,2413227)}; - const q_val hist3[] = {q_val(14,18878), q_val(21,8168), q_val(26,72634), q_val(32,2342292)}; + const q_val hist1[] = {q_val(6,21208), q_val(19,8227), q_val(32,73051), q_val(37,2339486)}; + const q_val hist2[] = {q_val(6,22647), q_val(19,9570), q_val(32,81839), q_val(37,2413227)}; + const q_val hist3[] = {q_val(6,18878), q_val(19,8168), q_val(32,72634), q_val(37,2342292)}; metrics.insert(metric_t(1, 1104, 1, to_vector(hist1))); metrics.insert(metric_t(1, 1106, 1, to_vector(hist2))); @@ -51,21 +50,38 @@ namespace illumina{ namespace interop { namespace unittest { const int tmp[] = { - 4,206, - 1,0,80,4,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,216,82,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,35,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,91 - ,29,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,158,178,35,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,0,82,4,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0,0,0,0,0,119,88,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,98,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,175,63,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0,171,210,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,0,80,4,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,190,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,232,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,186,27,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,148,189,35,0,0,0,0 - ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - ,0,0,0,0,0,0,0 + 4 + ,-50,1,0,80,4,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,-40,82,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,35,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,91,29,1,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-98,-78,35,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,1,0,82,4,1,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,119,88,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,98,37,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,-81,63,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,-85,-46,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,80,4,2,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,-66,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-24,31,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,-70,27,1,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,-108,-67,35,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + ,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; buffer.assign(tmp, tmp+util::length_of(tmp)); }/** Get the summary for these metrics @@ -80,23 +96,24 @@ namespace illumina{ namespace interop { namespace unittest const model::run::read_info reads[]={ model::run::read_info(1, 1, 3, false) }; + const float q30 = 98.803192138671875f; summary.initialize(to_vector(reads), lane_count, surface_count, channel_count); summary[0][0].tile_count(2); summary[0][0].projected_yield_g(0.0098816361278295517); summary[0][0].yield_g(0.0074112270958721638f); - summary[0][0].percent_gt_q30(95.733200073242188f); + summary[0][0].percent_gt_q30(q30); summary[0][0][0].tile_count(2); summary[0][0][0].projected_yield_g(0.0098816361278295517); summary[0][0][0].yield_g(0.0074112270958721638f); - summary[0][0][0].percent_gt_q30(95.733200073242188f); + summary[0][0][0].percent_gt_q30(q30); summary[0][0].cycle_state().qscored_cycle_range(model::run::cycle_range(1, 2)); summary[0].summary().projected_yield_g(0.0098816361278295517); summary[0].summary().yield_g(0.0074112270958721638f); - summary[0].summary().percent_gt_q30(95.733200073242188f); - summary.total_summary().percent_gt_q30(95.733200073242188f); + summary[0].summary().percent_gt_q30(q30); + summary.total_summary().percent_gt_q30(q30); summary.total_summary().projected_yield_g(0.0098816361278295517); summary.total_summary().yield_g(0.0074112270958721638f); - summary.nonindex_summary().percent_gt_q30(95.733200073242188f); + summary.nonindex_summary().percent_gt_q30(q30); summary.nonindex_summary().projected_yield_g(0.0098816361278295517); summary.nonindex_summary().yield_g(0.0074112270958721638f); summary.cycle_state().qscored_cycle_range(model::run::cycle_range(1, 2)); From e18247100c92578618d031133c57921b04b75e9c Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Fri, 16 Dec 2016 14:50:06 -0800 Subject: [PATCH 7/8] IPA-5871: Eliminate Swig warnings in C# --- interop/io/format/metric_format.h | 12 +- interop/io/plot/gnuplot.h | 6 +- interop/logic/metric/q_metric.h | 4 +- interop/logic/metric/tile_metric.h | 4 +- interop/model/metric_base/metric_set.h | 28 ++- interop/model/plot/data_point_collection.h | 17 +- interop/model/plot/heatmap_data.h | 23 ++- interop/model/plot/plot_data.h | 15 +- .../model/summary/index_flowcell_summary.h | 8 +- interop/model/summary/index_lane_summary.h | 8 +- interop/model/summary/lane_summary.h | 140 +++++++------- interop/model/summary/read_summary.h | 138 +++++++------- interop/model/summary/run_summary.h | 18 +- interop/model/table/imaging_table.h | 15 +- src/apps/dumpbin.cpp | 2 +- src/examples/example3.cpp | 2 +- src/examples/example_q_metric.cpp | 4 +- src/ext/swig/arrays/arrays_impl.i | 27 +-- src/ext/swig/metrics.i | 4 +- src/ext/swig/plot.i | 16 +- src/ext/swig/summary.i | 22 +-- src/ext/swig/table.i | 4 + src/ext/swig/util/operator_overload.i | 88 ++++++++- .../logic/metric/extraction_metric.cpp | 2 +- src/interop/logic/metric/q_metric.cpp | 8 +- src/tests/csharp/CMakeLists.txt | 1 + src/tests/csharp/CoreTests.cs | 180 ++++++++++++++++++ src/tests/csharp/logic/ImagingTableLogic.cs | 2 +- .../interop/logic/index_summary_test.cpp | 21 ++ .../interop/logic/plot_flowcell_test.cpp | 2 +- src/tests/interop/logic/plot_heatmap_test.cpp | 2 +- src/tests/interop/logic/plot_logic_test.cpp | 8 +- .../interop/metrics/tile_metrics_test.cpp | 29 +-- src/tests/interop/util/stat_test.cpp | 2 +- 34 files changed, 583 insertions(+), 279 deletions(-) create mode 100644 src/tests/csharp/CoreTests.cs diff --git a/interop/io/format/metric_format.h b/interop/io/format/metric_format.h index 02815a2b3..ff1414aaf 100644 --- a/interop/io/format/metric_format.h +++ b/interop/io/format/metric_format.h @@ -247,10 +247,10 @@ namespace illumina { namespace interop { namespace io { const size_t offset = metric_offset_map.size(); if(offset>= metric_set.size()) metric_set.resize(offset+1); - metric_set.at(offset).set_base(id); - count += Layout::map_stream(in, metric_set.at(offset), metric_set, true); + metric_set[offset].set_base(id); + count += Layout::map_stream(in, metric_set[offset], metric_set, true); if(!test_stream(in, metric_offset_map, count, record_size)) return; - if(Layout::skip_metric(metric_set.at(offset)))//Avoid adding control lanes in tile metrics + if(Layout::skip_metric(metric_set[offset]))//Avoid adding control lanes in tile metrics { metric_set.resize(offset); } @@ -259,9 +259,9 @@ namespace illumina { namespace interop { namespace io else { const size_t offset = metric_offset_map[metric.id()]; - INTEROP_ASSERTMSG(metric_set.at(offset).lane() != 0, offset); - count += Layout::map_stream(in, metric_set.at(offset), metric_set, false); - INTEROP_ASSERT(metric_set.at(offset).id()>0); + INTEROP_ASSERTMSG(metric_set[offset].lane() != 0, offset); + count += Layout::map_stream(in, metric_set[offset], metric_set, false); + INTEROP_ASSERT(metric_set[offset].id()>0); } } else diff --git a/interop/io/plot/gnuplot.h b/interop/io/plot/gnuplot.h index df99765dd..3c29a1348 100644 --- a/interop/io/plot/gnuplot.h +++ b/interop/io/plot/gnuplot.h @@ -53,7 +53,7 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t s = 0; s < swath_count; ++s) // Columns { - out << " " << table::handle_nan(data.at(x, y + s * data.tile_count())); + out << " " << table::handle_nan(data(x, y + s * data.tile_count())); } out << " nan"; } @@ -109,10 +109,10 @@ namespace illumina { namespace interop { namespace io { namespace plot out << "plot \"-\" matrix with image" << "\n"; for (size_t y = 0; y < data.column_count(); ++y) { - out << data.at(0, y); + out << data(0, y); for (size_t x = 1; x < data.row_count(); ++x) { - out << " " << table::handle_nan(data.at(x, y)); + out << " " << table::handle_nan(data(x, y)); } out << "\n"; } diff --git a/interop/logic/metric/q_metric.h b/interop/logic/metric/q_metric.h index 4f66931f7..403a4a894 100644 --- a/interop/logic/metric/q_metric.h +++ b/interop/logic/metric/q_metric.h @@ -160,7 +160,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric */ inline size_t count_qvals(const model::metric_base::metric_set& q_metric_set) { - return q_metric_set.size() > 0 ? q_metric_set.at(0).size() : 0; + return q_metric_set.size() > 0 ? q_metric_set[0].size() : 0; } /** Test whether the q-values are compressed * @@ -169,7 +169,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric */ inline size_t count_qvals( const model::metric_base::metric_set& q_metric_set) { - return q_metric_set.size() > 0 ? q_metric_set.at(0).size() : 0; + return q_metric_set.size() > 0 ? q_metric_set[0].size() : 0; } /** Test whether the q-values are compressed * diff --git a/interop/logic/metric/tile_metric.h b/interop/logic/metric/tile_metric.h index d45b93a06..68e05c1dd 100644 --- a/interop/logic/metric/tile_metric.h +++ b/interop/logic/metric/tile_metric.h @@ -61,7 +61,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric constants::tile_naming_method tile_naming_method_from_metric(const model::metric_base::metric_set& metric_set) { if(metric_set.size() == 0 ) return constants::UnknownTileNamingMethod; - return tile_naming_method_from_metric(metric_set.at(0)); + return tile_naming_method_from_metric(metric_set[0]); } /** Tile number * @@ -81,7 +81,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric */ inline ::uint32_t section(const ::uint32_t tile_id, const constants::tile_naming_method method) { - if(method == constants::FiveDigit) return (tile_id / 100) % 10; + if( method == constants::FiveDigit ) return (tile_id / 100) % 10; return std::numeric_limits< ::uint32_t >::min(); } /** Surface number diff --git a/interop/model/metric_base/metric_set.h b/interop/model/metric_base/metric_set.h index dfd2e35b2..a332a9464 100644 --- a/interop/model/metric_base/metric_set.h +++ b/interop/model/metric_base/metric_set.h @@ -314,21 +314,45 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * @param n index * @return metric */ + metric_type &operator[](const size_t n) throw(index_out_of_bounds_exception) + { + if(n >= m_data.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index out of bounds"); + return m_data[n]; + } + + /** Get a metric at the given index + * + * @param n index + * @return metric + */ + const metric_type &operator[](const size_t n)const throw(index_out_of_bounds_exception) + { + if(n >= m_data.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index out of bounds"); + return m_data[n]; + } + + /** Get a metric at the given index + * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) + * @param n index + * @return metric + */ metric_type &at(const size_t n) throw(index_out_of_bounds_exception) { if(n >= m_data.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index out of bounds"); - return m_data.at(n); + return m_data[n]; } /** Get a metric at the given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return metric */ const metric_type &at(const size_t n)const throw(index_out_of_bounds_exception) { if(n >= m_data.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index out of bounds"); - return m_data.at(n); + return m_data[n]; } /** Set the version of the InterOp file diff --git a/interop/model/plot/data_point_collection.h b/interop/model/plot/data_point_collection.h index 0434a48a0..2baccd1bb 100644 --- a/interop/model/plot/data_point_collection.h +++ b/interop/model/plot/data_point_collection.h @@ -46,7 +46,7 @@ namespace illumina { namespace interop { namespace model { namespace plot * @param index index of point * @return data point */ - const Point &at(const size_type index) const throw(model::index_out_of_bounds_exception) + const Point & operator[](const size_t index) const throw(model::index_out_of_bounds_exception) { if(index >= m_points.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Index out of bounds"); @@ -58,22 +58,23 @@ namespace illumina { namespace interop { namespace model { namespace plot * @param index index of point * @return data point */ - const_reference operator[](const size_type index) const throw(model::index_out_of_bounds_exception) + Point & operator[](const size_t index) throw(model::index_out_of_bounds_exception) { if(index >= m_points.size()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Index out of bounds"); + INTEROP_THROW( model::index_out_of_bounds_exception, "Index out of bounds"); return m_points[index]; } /** Get point at index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param index index of point * @return data point */ - reference operator[](const size_type index) throw(model::index_out_of_bounds_exception) + const Point &at(const size_t index) const throw(model::index_out_of_bounds_exception) { if(index >= m_points.size()) - INTEROP_THROW( model::index_out_of_bounds_exception, "Index out of bounds"); + INTEROP_THROW(model::index_out_of_bounds_exception, "Index out of bounds"); return m_points[index]; } @@ -81,7 +82,7 @@ namespace illumina { namespace interop { namespace model { namespace plot * * @return number of points in the collection */ - size_type size() const + size_t size() const { return m_points.size(); } @@ -91,7 +92,7 @@ namespace illumina { namespace interop { namespace model { namespace plot * @param n number of times to assign * @param val value */ - void assign(const size_type n, const Point &val) + void assign(const size_t n, const Point &val) { m_points.assign(n, val); } @@ -100,7 +101,7 @@ namespace illumina { namespace interop { namespace model { namespace plot * * @param n given size */ - void resize(const size_type n) + void resize(const size_t n) { m_points.resize(n); } diff --git a/interop/model/plot/heatmap_data.h b/interop/model/plot/heatmap_data.h index d6982cef2..8b38f8553 100644 --- a/interop/model/plot/heatmap_data.h +++ b/interop/model/plot/heatmap_data.h @@ -86,6 +86,7 @@ namespace illumina { namespace interop { namespace model { namespace plot * * TODO: This should thrown an exception if wrong * + * @deprecated Will be removed in next feature version (use operator() instead for C++ Code) * @param row row index * @param col column index * @return value @@ -104,6 +105,7 @@ namespace illumina { namespace interop { namespace model { namespace plot /** Get value at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param idx index * @return value */ @@ -115,6 +117,19 @@ namespace illumina { namespace interop { namespace model { namespace plot INTEROP_ASSERT(m_data != 0); return m_data[idx]; } + /** Get value at given index + * + * @param idx index + * @return value + */ + float operator[](const size_t idx) const throw(model::index_out_of_bounds_exception) + { + if (idx >= length()) + INTEROP_THROW(model::index_out_of_bounds_exception, "Index out of bounds"); + INTEROP_ASSERT(idx < m_num_rows*m_num_columns); + INTEROP_ASSERT(m_data != 0); + return m_data[idx]; + } /** Get value at given row and column * @@ -124,12 +139,12 @@ namespace illumina { namespace interop { namespace model { namespace plot * @param col column index * @return value */ - float &operator()(const size_t row, const size_t col) throw(model::index_out_of_bounds_exception) + float operator()(const size_t row, const size_t col) const throw(model::index_out_of_bounds_exception) { if (row >= m_num_rows) INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); if (col >= m_num_columns) - INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds (operator): " << col << " >= " << m_num_columns); + INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds (operator const): " << col << " >= " << m_num_columns); const size_t idx = index_of(row, col); INTEROP_ASSERT(idx < m_num_rows*m_num_columns); INTEROP_ASSERT(m_data != 0); @@ -144,12 +159,12 @@ namespace illumina { namespace interop { namespace model { namespace plot * @param col column index * @return value */ - float operator()(const size_t row, const size_t col) const throw(model::index_out_of_bounds_exception) + float &operator()(const size_t row, const size_t col) throw(model::index_out_of_bounds_exception) { if (row >= m_num_rows) INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); if (col >= m_num_columns) - INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds (operator const): " << col << " >= " << m_num_columns); + INTEROP_THROW(model::index_out_of_bounds_exception, "Column index out of bounds (operator): " << col << " >= " << m_num_columns); const size_t idx = index_of(row, col); INTEROP_ASSERT(idx < m_num_rows*m_num_columns); INTEROP_ASSERT(m_data != 0); diff --git a/interop/model/plot/plot_data.h b/interop/model/plot/plot_data.h index 449f85a8f..97bb88a64 100644 --- a/interop/model/plot/plot_data.h +++ b/interop/model/plot/plot_data.h @@ -44,7 +44,7 @@ namespace illumina { namespace interop { namespace model { namespace plot { * * @param n given size */ - void resize(const size_type n) + void resize(const size_t n) { m_series.resize(n); } @@ -53,7 +53,7 @@ namespace illumina { namespace interop { namespace model { namespace plot { * @param n given size * @param val value to assign */ - void assign(const size_type n, const series& val) + void assign(const size_t n, const series& val) { m_series.assign(n, val); } @@ -67,21 +67,22 @@ namespace illumina { namespace interop { namespace model { namespace plot { } /** Get point at index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param index index of point * @return data point */ - const series& at(const size_type index)const throw(model::index_out_of_bounds_exception) + const series& at(const size_t index)const throw(model::index_out_of_bounds_exception) { if(index >= m_series.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); - return m_series.at(index); + return m_series[index]; } /** Get point at index * * @param index index of point * @return data point */ - const_reference operator[](const size_type index)const throw(model::index_out_of_bounds_exception) + series& operator[](const size_t index) throw(model::index_out_of_bounds_exception) { if(index >= m_series.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); @@ -92,7 +93,7 @@ namespace illumina { namespace interop { namespace model { namespace plot { * @param index index of point * @return data point */ - reference operator[](const size_type index) throw(model::index_out_of_bounds_exception) + const series& operator[](const size_t index)const throw(model::index_out_of_bounds_exception) { if(index >= m_series.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Row index out of bounds"); @@ -102,7 +103,7 @@ namespace illumina { namespace interop { namespace model { namespace plot { * * @return number of points in the collection */ - size_type size()const + size_t size()const { return m_series.size(); } diff --git a/interop/model/summary/index_flowcell_summary.h b/interop/model/summary/index_flowcell_summary.h index 58f47e2ce..51178accf 100644 --- a/interop/model/summary/index_flowcell_summary.h +++ b/interop/model/summary/index_flowcell_summary.h @@ -57,7 +57,7 @@ namespace illumina { namespace interop { namespace model { namespace summary * @param n index * @return reference to lane summary */ - reference operator[](const size_type n) throw( model::index_out_of_bounds_exception ) + index_lane_summary& operator[](const size_t n) throw( model::index_out_of_bounds_exception ) { if(n >= m_lane_summaries.size()) INTEROP_THROW(index_out_of_bounds_exception, "Lane index exceeds lane count"); @@ -68,7 +68,7 @@ namespace illumina { namespace interop { namespace model { namespace summary * @param n index * @return constant reference to lane summary */ - const_reference operator[](const size_type n)const throw( model::index_out_of_bounds_exception ) + const index_lane_summary& operator[](const size_t n)const throw( model::index_out_of_bounds_exception ) { if(n >= m_lane_summaries.size()) INTEROP_THROW(index_out_of_bounds_exception, "Lane index exceeds lane count"); @@ -76,10 +76,11 @@ namespace illumina { namespace interop { namespace model { namespace summary } /** Get reference to lane summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return reference to lane summary */ - index_lane_summary& at(const size_type n) throw( model::index_out_of_bounds_exception ) + index_lane_summary& at(const size_t n) throw( model::index_out_of_bounds_exception ) { if(n >= m_lane_summaries.size()) INTEROP_THROW(index_out_of_bounds_exception, "Lane index exceeds lane count"); @@ -87,6 +88,7 @@ namespace illumina { namespace interop { namespace model { namespace summary } /** Get constant reference to lane summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return constant reference to lane summary */ diff --git a/interop/model/summary/index_lane_summary.h b/interop/model/summary/index_lane_summary.h index 3da11ccb7..88b8656f2 100644 --- a/interop/model/summary/index_lane_summary.h +++ b/interop/model/summary/index_lane_summary.h @@ -91,7 +91,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { * @param n index * @return reference to lane summary */ - reference operator[](const size_type n) throw( model::index_out_of_bounds_exception ) + index_count_summary& operator[](const size_type n) throw( model::index_out_of_bounds_exception ) { if(n >= m_count_summaries.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index sequence index exceeds index sequence count"); @@ -102,7 +102,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { * @param n index * @return constant reference to lane summary */ - const_reference operator[](const size_type n)const throw( model::index_out_of_bounds_exception ) + const index_count_summary& operator[](const size_t n)const throw( model::index_out_of_bounds_exception ) { if(n >= m_count_summaries.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index sequence index exceeds index sequence count"); @@ -110,10 +110,11 @@ namespace illumina { namespace interop { namespace model { namespace summary { } /** Get reference to lane summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return reference to lane summary */ - index_count_summary& at(const size_type n) throw( model::index_out_of_bounds_exception ) + index_count_summary& at(const size_t n) throw( model::index_out_of_bounds_exception ) { if(n >= m_count_summaries.size()) INTEROP_THROW(index_out_of_bounds_exception, "Index sequence index exceeds index sequence count"); @@ -121,6 +122,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { } /** Get constant reference to lane summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return constant reference to lane summary */ diff --git a/interop/model/summary/lane_summary.h b/interop/model/summary/lane_summary.h index c3bba91a3..0f72b0e8a 100644 --- a/interop/model/summary/lane_summary.h +++ b/interop/model/summary/lane_summary.h @@ -55,80 +55,13 @@ namespace illumina { namespace interop { namespace model { namespace summary { } - public: - /** @defgroup lane_summary Read/Lane summary - * - * Summary of key metrics in each read/lane - * - * @ingroup read_summary - * @ref illumina::interop::model::summary::lane_summary "See full class description" - * @ref illumina::interop::model::summary::stat_summary "See full class description" - * @{ - */ - /** Get constant reference to lane_summary at given index - * - * @param n index - * @return constant reference to lane_summary - */ - const_reference operator[](const size_type n) const throw(model::index_out_of_bounds_exception) - { - if (n >= m_summary_by_surface.size()) - INTEROP_THROW( index_out_of_bounds_exception, "Surface index exceeds surface count"); - return m_summary_by_surface[n]; - } - - /** Get constant reference to lane_summary at given index - * - * @param n index - * @return constant reference to lane_summary - */ - const_reference at(const size_type n) const throw(model::index_out_of_bounds_exception) - { - if (n >= m_summary_by_surface.size()) - INTEROP_THROW( index_out_of_bounds_exception, "Surface index exceeds surface count"); - return m_summary_by_surface[n]; - } - /** Get lane number - * - * @return lane number - */ - size_t lane() const - { - return m_lane; - } - /** Get number of tiles in the lane - * - * @return number of tiles in the lane - */ - size_t tile_count() const - { - return m_tile_count; - } - /** Get statistics summarizing the cycle of each RTA state of tiles in the lane - * - * @return statistics summarizing the cycle of each RTA state of tiles in the lane - */ - const cycle_state_summary &cycle_state() const - { - return m_cycle_state; - } - /** Get number of summaries by surface - * - * @return number of summaries by surface - */ - size_t size() const - { - return m_summary_by_surface.size(); - } - /** @} */ - public: /** Get reference to lane_summary at given index * * @param n index * @return reference to lane_summary */ - reference operator[](const size_type n) throw(model::index_out_of_bounds_exception) + surface_summary & operator[](const size_t n) throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_surface.size()) INTEROP_THROW( index_out_of_bounds_exception, "Surface index exceeds surface count"); @@ -137,10 +70,11 @@ namespace illumina { namespace interop { namespace model { namespace summary /** Get reference to lane_summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return reference to lane_summary */ - surface_summary &at(const size_type n) throw(model::index_out_of_bounds_exception) + surface_summary &at(const size_t n) throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_surface.size()) INTEROP_THROW( index_out_of_bounds_exception, "Surface index exceeds surface count"); @@ -216,6 +150,74 @@ namespace illumina { namespace interop { namespace model { namespace summary return m_cycle_state; } + public: + /** @defgroup lane_summary Read/Lane summary + * + * Summary of key metrics in each read/lane + * + * @ingroup read_summary + * @ref illumina::interop::model::summary::lane_summary "See full class description" + * @ref illumina::interop::model::summary::stat_summary "See full class description" + * @{ + */ + /** Get constant reference to lane_summary at given index + * + * @param n index + * @return constant reference to lane_summary + */ + const surface_summary& operator[](const size_t n) const throw(model::index_out_of_bounds_exception) + { + if (n >= m_summary_by_surface.size()) + INTEROP_THROW( index_out_of_bounds_exception, "Surface index exceeds surface count"); + return m_summary_by_surface[n]; + } + + /** Get constant reference to lane_summary at given index + * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) + * @param n index + * @return constant reference to lane_summary + */ + const_reference at(const size_type n) const throw(model::index_out_of_bounds_exception) + { + if (n >= m_summary_by_surface.size()) + INTEROP_THROW( index_out_of_bounds_exception, "Surface index exceeds surface count"); + return m_summary_by_surface[n]; + } + /** Get lane number + * + * @return lane number + */ + size_t lane() const + { + return m_lane; + } + /** Get number of tiles in the lane + * + * @return number of tiles in the lane + */ + size_t tile_count() const + { + return m_tile_count; + } + /** Get statistics summarizing the cycle of each RTA state of tiles in the lane + * + * @return statistics summarizing the cycle of each RTA state of tiles in the lane + */ + const cycle_state_summary &cycle_state() const + { + return m_cycle_state; + } + /** Get number of summaries by surface + * + * @return number of summaries by surface + */ + size_t size() const + { + return m_summary_by_surface.size(); + } + /** @} */ + private: size_t m_lane; size_t m_tile_count; diff --git a/interop/model/summary/read_summary.h b/interop/model/summary/read_summary.h index 1310a72c2..326527a80 100644 --- a/interop/model/summary/read_summary.h +++ b/interop/model/summary/read_summary.h @@ -49,80 +49,63 @@ namespace illumina { namespace interop { namespace model { namespace summary } public: - /** @defgroup read_summary Read summary - * - * Summary of key metrics in each read - * - * @ingroup run_summary - * @ref illumina::interop::model::summary::read_summary "See full class description" - * @{ - */ - /** Get constant reference to lane_summary at given index + /** Get reference to lane_summary at given index * * @param n index - * @return constant reference to lane_summary + * @return reference to lane_summary */ - const_reference operator[](const size_type n) const throw(model::index_out_of_bounds_exception) + lane_summary & operator[](const size_type n) throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_lane.size()) INTEROP_THROW( index_out_of_bounds_exception, "Lane index exceeds lane count"); return m_summary_by_lane[n]; } - /** Get constant reference to lane_summary at given index + /** Get reference to lane_summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index - * @return constant reference to lane_summary + * @return reference to lane_summary */ - const_reference at(const size_type n) const throw(model::index_out_of_bounds_exception) + lane_summary &at(const size_type n) throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_lane.size()) INTEROP_THROW( index_out_of_bounds_exception, "Lane index exceeds lane count"); return m_summary_by_lane[n]; } - /** Get number of summaries by lane - * - * @return number of summaries by lane - */ - size_t size() const - { - return m_summary_by_lane.size(); - } - - /** Get number of lanes + /** Get random access iterator to start of summaries by lane * - * @return number of lanes + * @return random access iterator */ - size_t lane_count() const + iterator begin() { - return m_summary_by_lane.size(); + return m_summary_by_lane.begin(); } - /** Get constant random access iterator to start of summaries by lane + /** Get random access iterator to end of summaries by lane * - * @return constant random access iterator + * @return random access iterator */ - const_iterator begin() const + iterator end() { - return m_summary_by_lane.begin(); + return m_summary_by_lane.end(); } - /** Get constant random access iterator to end of summaries by lane + /** Resize the summary by lane vector * - * @return constant random access iterator + * @param n new size of summary by lane vector */ - const_iterator end() const + void resize(const size_type n) { - return m_summary_by_lane.end(); + m_summary_by_lane.resize(n); } - public: /** Get read information * * @return read information */ - const run::read_info &read() const + run::read_info &read() { return m_read; } @@ -131,69 +114,96 @@ namespace illumina { namespace interop { namespace model { namespace summary * * @return summary metrics */ - const metric_summary &summary() const + metric_summary &summary() { return m_metric_summary; } - /** @} */ + + /** Set the summary + * + * @param summary metric summary + */ + void summary(const metric_summary &summary) + { + m_metric_summary = summary; + } public: - /** Get reference to lane_summary at given index + /** @defgroup read_summary Read summary + * + * Summary of key metrics in each read + * + * @ingroup run_summary + * @ref illumina::interop::model::summary::read_summary "See full class description" + * @{ + */ + /** Get constant reference to lane_summary at given index * * @param n index - * @return reference to lane_summary + * @return constant reference to lane_summary */ - reference operator[](const size_type n) throw(model::index_out_of_bounds_exception) + const lane_summary& operator[](const size_t n) const throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_lane.size()) INTEROP_THROW( index_out_of_bounds_exception, "Lane index exceeds lane count"); return m_summary_by_lane[n]; } - /** Get reference to lane_summary at given index + /** Get constant reference to lane_summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index - * @return reference to lane_summary + * @return constant reference to lane_summary */ - lane_summary &at(const size_type n) throw(model::index_out_of_bounds_exception) + const_reference at(const size_type n) const throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_lane.size()) INTEROP_THROW( index_out_of_bounds_exception, "Lane index exceeds lane count"); return m_summary_by_lane[n]; } - /** Get random access iterator to start of summaries by lane + /** Get number of summaries by lane * - * @return random access iterator + * @return number of summaries by lane */ - iterator begin() + size_t size() const { - return m_summary_by_lane.begin(); + return m_summary_by_lane.size(); } - /** Get random access iterator to end of summaries by lane + /** Get number of lanes * - * @return random access iterator + * @return number of lanes */ - iterator end() + size_t lane_count() const { - return m_summary_by_lane.end(); + return m_summary_by_lane.size(); } - /** Resize the summary by lane vector + /** Get constant random access iterator to start of summaries by lane * - * @param n new size of summary by lane vector + * @return constant random access iterator */ - void resize(const size_type n) + const_iterator begin() const { - m_summary_by_lane.resize(n); + return m_summary_by_lane.begin(); } + /** Get constant random access iterator to end of summaries by lane + * + * @return constant random access iterator + */ + const_iterator end() const + { + return m_summary_by_lane.end(); + } + + public: /** Get read information * * @return read information */ - run::read_info &read() + const run::read_info &read() const { return m_read; } @@ -202,19 +212,11 @@ namespace illumina { namespace interop { namespace model { namespace summary * * @return summary metrics */ - metric_summary &summary() + const metric_summary &summary() const { return m_metric_summary; } - - /** Set the summary - * - * @param summary metric summary - */ - void summary(const metric_summary &summary) - { - m_metric_summary = summary; - } + /** @} */ private: /** Read information */ diff --git a/interop/model/summary/run_summary.h b/interop/model/summary/run_summary.h index 5dc75906f..8b39704ac 100644 --- a/interop/model/summary/run_summary.h +++ b/interop/model/summary/run_summary.h @@ -147,15 +147,15 @@ namespace illumina { namespace interop { namespace model { namespace summary b->summary().resize(m_channel_count); for (size_t lane = 0; lane < m_lane_count; ++lane) { - b->at(lane).lane(lane + 1); - b->at(lane).resize_stat(m_channel_count); + b->operator[](lane).lane(lane + 1); + b->operator[](lane).resize_stat(m_channel_count); if(m_surface_count > 1) { - b->at(lane).resize(m_surface_count); + b->operator[](lane).resize(m_surface_count); for (size_t surface = 0; surface < m_surface_count; ++surface) { - b->at(lane).at(surface).surface(surface + 1); - b->at(lane).at(surface).resize_stat(m_channel_count); + b->operator[](lane)[surface].surface(surface + 1); + b->operator[](lane)[surface].resize_stat(m_channel_count); } } } @@ -176,7 +176,7 @@ namespace illumina { namespace interop { namespace model { namespace summary * @param n index * @return reference to read_summary */ - reference operator[](const size_type n) throw(model::index_out_of_bounds_exception) + read_summary & operator[](const size_t n) throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_read.size()) INTEROP_THROW(index_out_of_bounds_exception, "Read index exceeds read count"); @@ -188,7 +188,7 @@ namespace illumina { namespace interop { namespace model { namespace summary * @param n index * @return constant reference to read_summary */ - const_reference operator[](const size_type n) const throw(model::index_out_of_bounds_exception) + const read_summary & operator[](const size_t n) const throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_read.size()) INTEROP_THROW(index_out_of_bounds_exception, "Read index exceeds read count"); @@ -197,10 +197,11 @@ namespace illumina { namespace interop { namespace model { namespace summary /** Get reference to read_summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return reference to read_summary */ - read_summary &at(const size_type n) throw(model::index_out_of_bounds_exception) + read_summary &at(const size_t n) throw(model::index_out_of_bounds_exception) { if (n >= m_summary_by_read.size()) INTEROP_THROW(index_out_of_bounds_exception, "Read index exceeds read count"); @@ -209,6 +210,7 @@ namespace illumina { namespace interop { namespace model { namespace summary /** Get constant reference to read_summary at given index * + * @deprecated Will be removed in next feature version (use operator[] instead for C++ Code) * @param n index * @return constant reference to read_summary */ diff --git a/interop/model/table/imaging_table.h b/interop/model/table/imaging_table.h index 19b18e5d7..ba44f3fdf 100644 --- a/interop/model/table/imaging_table.h +++ b/interop/model/table/imaging_table.h @@ -59,14 +59,14 @@ namespace illumina { namespace interop { namespace model { namespace table * @param subcol sub column offset * @return cell value */ - float operator()(const size_t r, const column_id c, const size_t subcol=0)const + float operator()(const size_t r, const column_id c, const size_t subcol=0)const throw(model::index_out_of_bounds_exception) { if(static_cast(c) >= m_enum_to_index.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Invalid enum id for column"); const size_t col = m_enum_to_index[static_cast(c)]; if(col >= m_enum_to_index.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Invalid enum - column not filled"); - return at(r, col, subcol); + return operator()(r, col, subcol); } /** Get a reference to a row * @@ -75,12 +75,19 @@ namespace illumina { namespace interop { namespace model { namespace table * @param subcol sub column offset * @return cell value */ - float operator()(const size_t r, const size_t c, const size_t subcol=0)const + float operator()(const size_t r, const size_t c, const size_t subcol=0)const throw(model::index_out_of_bounds_exception) { - return at(r, c, subcol); + if(r >=m_row_count) INTEROP_THROW(model::index_out_of_bounds_exception, "Row out of bounds"); + if(c >=m_columns.size()) INTEROP_THROW(model::index_out_of_bounds_exception, "Column out of bounds"); + const size_t col = m_columns[c].offset()+subcol; + if(col >=m_col_count) INTEROP_THROW(model::index_out_of_bounds_exception, "Column data offset out of bounds"); + const size_t index = r*m_col_count+col; + INTEROP_ASSERT(index < m_data.size()); + return m_data[index]; } /** Get a reference to a row * + * @deprecated Will be removed in next feature version (use operator() instead for C++ Code) * @param r row row index * @param c col column index * @param subcol sub column offset diff --git a/src/apps/dumpbin.cpp b/src/apps/dumpbin.cpp index 7aa2faf14..e9f67177b 100644 --- a/src/apps/dumpbin.cpp +++ b/src/apps/dumpbin.cpp @@ -119,7 +119,7 @@ struct subset_copier const size_t total = std::min(m_total, metrics.size()); for(size_t i=0;i().insert(metrics.at(i)); + m_run.get().insert(metrics[i]); } private: run_metrics& m_run; diff --git a/src/examples/example3.cpp b/src/examples/example3.cpp index 8931117d2..e57f177dc 100644 --- a/src/examples/example3.cpp +++ b/src/examples/example3.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) try { - std::time_t t = static_cast(extraction_metric_set.at(0).date_time()); + std::time_t t = static_cast(extraction_metric_set[0].date_time()); std::tm *tm = std::gmtime(&t); if (tm != 0) { diff --git a/src/examples/example_q_metric.cpp b/src/examples/example_q_metric.cpp index 9d4efff4e..2420d6ac4 100644 --- a/src/examples/example_q_metric.cpp +++ b/src/examples/example_q_metric.cpp @@ -57,14 +57,14 @@ int main(int argc, char** argv) { // @ [Calculating Total >= Q30] - q_metric &metric0 = q_metric_set.at(0); + q_metric &metric0 = q_metric_set[0]; std::cout << "Total >= Q30: " << metric0.total_over_qscore(30, q_metric_set.bins()) << std::endl; // @ [Calculating Total >= Q30] // @ [Calculating Percent >= Q30] - q_metric &metric1 = q_metric_set.at(0); + q_metric &metric1 = q_metric_set[0]; std::cout << "Percent >= Q30: " << metric1.percent_over_qscore(30, q_metric_set.bins()) << std::endl; // @ [Calculating Percent >= Q30] diff --git a/src/ext/swig/arrays/arrays_impl.i b/src/ext/swig/arrays/arrays_impl.i index a499a2616..ec20a15c8 100644 --- a/src/ext/swig/arrays/arrays_impl.i +++ b/src/ext/swig/arrays/arrays_impl.i @@ -2,35 +2,10 @@ #if defined(SWIGCSHARP) %include "arrays_csharp_impl.i" #elif defined(SWIGJAVA) - -%include "arrays_java.i" - -%apply signed char[] { uint8_t *buffer};// Allow unsigned char to be wrapped as a byte -%apply (char *STRING, size_t LENGTH) { (uint8_t *buffer, size_t buffer_size) }; -%apply int[] {uint16_t *} -%apply long[] {uint32_t *} -%apply long[] {uint64_t *} - -%apply float[] {float *}; - +%include "arrays_java_impl.i" #elif defined(SWIGPYTHON) %include "arrays_numpy_impl.i" #endif -// Collection of methods we need to ignore in a common vector interface -// Collection of typedefs we need to make SWIG aware of in a common vector interface -%define WRAP_VECTOR(vector_t) -%ignore vector_t::iterator; -%ignore vector_t::begin; -%ignore vector_t::end; -%ignore vector_t::operator[]; -%apply size_t { vector_t::size_type }; -%enddef - -%define WRAP_AS_VECTOR(value_t) -%apply value_t& { std::vector::reference }; -//%apply value_t& const { std::vector::const_reference }; -WRAP_VECTOR(std::vector); -%enddef diff --git a/src/ext/swig/metrics.i b/src/ext/swig/metrics.i index c3a32c532..c45f2086c 100644 --- a/src/ext/swig/metrics.i +++ b/src/ext/swig/metrics.i @@ -64,15 +64,12 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) using namespace illumina::interop::model::metrics; namespace metric_base = illumina::interop::model::metric_base; - %ignore illumina::interop::model::metric_base::metric_set::at(const size_t)const; %ignore illumina::interop::model::metric_base::metric_set::populate_tile_numbers_for_lane; %ignore illumina::interop::model::metric_base::metric_set::populate_tile_numbers_for_lane_surface; %ignore illumina::interop::model::metric_base::metric_set::offset_map; - //%ignore illumina::interop::model::metric_base::metric_set::offset_map; TODO Ignore functions that take set are argument %apply size_t { std::map< std::size_t, metric_t >::size_type }; %apply uint64_t { metric_base::metric_set::id_t }; - WRAP_VECTOR(illumina::interop::model::metric_base::metric_set); %enddef @@ -123,6 +120,7 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) // Wrap vectors //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + %template(index_info_vector) std::vector< illumina::interop::model::metrics::index_info >; %template(tile_metric_map) std::map< uint64_t, illumina::interop::model::metric_base::base_metric >; diff --git a/src/ext/swig/plot.i b/src/ext/swig/plot.i index b2b8e8e00..545410f9a 100644 --- a/src/ext/swig/plot.i +++ b/src/ext/swig/plot.i @@ -44,7 +44,6 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) // Plotting //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -%ignore illumina::interop::model::plot::flowcell_data::tile_id(size_t const,size_t const); %{ #include "interop/model/plot/axes.h" #include "interop/model/plot/candle_stick_point.h" @@ -59,6 +58,8 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) #include "interop/model/plot/flowcell_data.h" %} +%ignore illumina::interop::model::plot::flowcell_data::tile_id(size_t const,size_t const); +RENAME_TEMPLATE_OPERATOR_CONST(illumina::interop::model::plot::heatmap_data); %ignore illumina::interop::model::plot::filter_options::option_iterator; %include "interop/model/plot/axes.h" @@ -74,28 +75,27 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %apply float { illumina::interop::model::plot::data_point::y_type }; %template(float_point) illumina::interop::model::plot::data_point; - %include "interop/model/plot/candle_stick_point.h" %include "interop/model/plot/bar_point.h" -WRAP_VECTOR(illumina::interop::model::plot::data_point_collection); -WRAP_VECTOR(illumina::interop::model::plot::data_point_collection); %include "interop/model/plot/data_point_collection.h" +WRAP_VECTOR(std::vector) %template(bar_vector) std::vector; +WRAP_VECTOR(std::vector) %template(candle_stick_vector) std::vector; + %template(candle_stick_collection) illumina::interop::model::plot::data_point_collection; %template(bar_collection) illumina::interop::model::plot::data_point_collection; %include "interop/model/plot/series.h" -WRAP_VECTOR(illumina::interop::model::plot::series); -WRAP_VECTOR(illumina::interop::model::plot::series); + %template(candle_stick_series) illumina::interop::model::plot::series; %template(bar_series) illumina::interop::model::plot::series; %include "interop/model/plot/plot_data.h" -WRAP_VECTOR(illumina::interop::model::plot::plot_data); -WRAP_VECTOR(illumina::interop::model::plot::plot_data); + %template(candle_stick_plot_data) illumina::interop::model::plot::plot_data; + %template(bar_plot_data) illumina::interop::model::plot::plot_data; diff --git a/src/ext/swig/summary.i b/src/ext/swig/summary.i index e4fc9316e..48c4b9d1c 100644 --- a/src/ext/swig/summary.i +++ b/src/ext/swig/summary.i @@ -53,10 +53,12 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) #include "interop/logic/metric/q_metric.h" %} -WRAP_VECTOR(illumina::interop::model::summary::lane_summary); -WRAP_VECTOR(illumina::interop::model::summary::read_summary); %ignore illumina::interop::model::summary::read_summary::read()const; -WRAP_VECTOR(illumina::interop::model::summary::run_summary); +%ignore illumina::interop::model::summary::lane_summary::cycle_state()const; +%ignore illumina::interop::model::summary::read_summary::summary()const; +%ignore illumina::interop::model::summary::run_summary::total_summary()const; +%ignore illumina::interop::model::summary::run_summary::nonindex_summary()const; +%ignore illumina::interop::model::summary::run_summary::cycle_state()const; %include "interop/model/summary/cycle_state_summary.h" %include "interop/model/summary/stat_summary.h" @@ -71,12 +73,6 @@ WRAP_VECTOR(illumina::interop::model::summary::run_summary); // Setup typemaps for summary metrics // -%ignore illumina::interop::model::summary::lane_summary::at(const size_type) const; -%ignore illumina::interop::model::summary::read_summary::at(const size_type) const; -WRAP_AS_VECTOR(illumina::interop::model::summary::surface_summary); -WRAP_AS_VECTOR(illumina::interop::model::summary::lane_summary); -WRAP_AS_VECTOR(illumina::interop::model::summary::read_summary); - %template(surface_summary_vector) std::vector; %template(lane_summary_vector) std::vector; @@ -93,6 +89,7 @@ WRAP_AS_VECTOR(illumina::interop::model::summary::read_summary); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Index summary model //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + %{ #include "interop/model/summary/index_count_summary.h" #include "interop/model/summary/index_lane_summary.h" @@ -100,14 +97,7 @@ WRAP_AS_VECTOR(illumina::interop::model::summary::read_summary); %} %include "interop/model/summary/index_count_summary.h" - -WRAP_AS_VECTOR(illumina::interop::model::summary::index_count_summary); -WRAP_VECTOR(illumina::interop::model::summary::index_lane_summary); %include "interop/model/summary/index_lane_summary.h" - - -WRAP_VECTOR(illumina::interop::model::summary::index_flowcell_summary); -WRAP_AS_VECTOR(illumina::interop::model::summary::index_lane_summary); %include "interop/model/summary/index_flowcell_summary.h" %template(index_count_summary_vector) std::vector; diff --git a/src/ext/swig/table.i b/src/ext/swig/table.i index 84368cb85..8051ef072 100644 --- a/src/ext/swig/table.i +++ b/src/ext/swig/table.i @@ -29,6 +29,7 @@ using Illumina.InterOp.Metrics; using Illumina.InterOp.Run; %} +// Ensure the C++ shared library is loaded by the Java interface file %pragma(java) jniclasscode=%{ static { System.loadLibrary("interop_table"); @@ -40,6 +41,7 @@ WRAP_METRICS(IMPORT_METRIC_WRAPPER) // This allows exceptions to be imported, but not belong to the module EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) +// Map an ID to an offset in a table %template(map_id_offset) std::map; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -52,6 +54,8 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %include "interop/model/table/imaging_column.h" %include "interop/model/table/imaging_table.h" + + %template(imaging_column_vector) std::vector< illumina::interop::model::table::imaging_column >; diff --git a/src/ext/swig/util/operator_overload.i b/src/ext/swig/util/operator_overload.i index e43098129..b58f4fc03 100644 --- a/src/ext/swig/util/operator_overload.i +++ b/src/ext/swig/util/operator_overload.i @@ -1,6 +1,21 @@ // Suppress warnings and expose operators to client languages // Most languages do no support operator overloading like C++, so these need to be renamed or ignored. +%ignore *::operator[](size_t const)const; +%ignore *::operator()const; +%ignore *::operator()(size_t const,size_t const)const; +%ignore *::operator()(size_t const,size_t const,size_t const)const; +%ignore *::operator()(size_t const,size_t const,size_t const,size_t const)const; +%ignore *::at; // .at is deprecated in the C++ code +%ignore operator enum_t; +%ignore operator<<; +%ignore operator>>; +%ignore *::iterator; +%ignore *::begin; +%ignore *::end; + + + #if defined(SWIGCSHARP) || defined(SWIGJAVA) || defined(SWIGPYTHON) // TODO: Add python section and rename to appropriate python operator @@ -10,19 +25,74 @@ %rename(is_less) operator< const; %rename(copy) operator=; %rename(to_int) operator uint64_t; + %rename(at) *::operator[]; + %rename(at) *::operator(); +//#elif defined(SWIGPYTHON) + /*%rename(__add__) *::operator+; +%rename(__pos__) *::operator+(); +%rename(__pos__) *::operator+() const; +%rename(__sub__) *::operator-; +%rename(__neg__) *::operator-() const; +%rename(__neg__) *::operator-(); +%rename(__mul__) *::operator*; +%rename(__deref__) *::operator*(); +%rename(__deref__) *::operator*() const; +%rename(__div__) *::operator/; +%rename(__mod__) *::operator%; +%rename(__logxor__) *::operator^; +%rename(__logand__) *::operator&; +%rename(__logior__) *::operator|; +%rename(__lognot__) *::operator~(); +%rename(__lognot__) *::operator~() const; +%rename(__not__) *::operator!(); +%rename(__not__) *::operator!() const; +%rename(__assign__) *::operator=; +%rename(__add_assign__) *::operator+=; +%rename(__sub_assign__) *::operator-=; +%rename(__mul_assign__) *::operator*=; +%rename(__div_assign__) *::operator/=; +%rename(__mod_assign__) *::operator%=; +%rename(__logxor_assign__) *::operator^=; +%rename(__logand_assign__) *::operator&=; +%rename(__logior_assign__) *::operator|=; +%rename(__lshift__) *::operator<<; +%rename(__lshift_assign__) *::operator<<=; +%rename(__rshift__) *::operator>>; +%rename(__rshift_assign__) *::operator>>=; +%rename(__eq__) *::operator==; +%rename(__ne__) *::operator!=; +%rename(__lt__) *::operator<; +%rename(__gt__) *::operator>; +%rename(__lte__) *::operator<=; +%rename(__gte__) *::operator>=; +%rename(__and__) *::operator&&; +%rename(__or__) *::operator||; +%rename(__preincr__) *::operator++(); +%rename(__postincr__) *::operator++(int); +%rename(__predecr__) *::operator--(); +%rename(__postdecr__) *::operator--(int); +%rename(__comma__) *::operator,(); +%rename(__comma__) *::operator,() const; +%rename(__member_ref__) *::operator->; +%rename(__member_func_ref__) *::operator->*; +%rename(__funcall__) *::operator(); +%rename(__aref__) *::operator[];*/ #else // Do nothing #endif -%ignore operator<<; -%ignore operator>>; -%ignore *::operator() const; - -// TODO: Remove at in C++ and rename operator[] to at -%ignore *::operator[]; -%ignore operator enum_t; - - +%define WRAP_VECTOR(vector_t) +%apply size_t { vector_t::size_type }; +%enddef +%define RENAME_TEMPLATE_OPERATOR_CONST(vector_t) // Prefer constant over the default, non-const +%ignore vector_t::at; +%ignore vector_t::operator[](size_t const); +%ignore vector_t::operator(); +%ignore vector_t::operator()(size_t const,size_t const); +%ignore vector_t::operator()(size_t const,size_t const,size_t const,size_t const); +%rename(at) vector_t::operator[]const; +%rename(at) vector_t::operator()const; +%enddef diff --git a/src/interop/logic/metric/extraction_metric.cpp b/src/interop/logic/metric/extraction_metric.cpp index 1451048c6..7320ef80e 100644 --- a/src/interop/logic/metric/extraction_metric.cpp +++ b/src/interop/logic/metric/extraction_metric.cpp @@ -18,7 +18,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric typedef model::metric_base::metric_set::const_iterator const_iterator; if(metrics.size()==0)return; if(n < metrics.size()) INTEROP_THROW(model::invalid_parameter, "Buffer size too small for metric set"); - if(channel >= metrics.at(0).channel_count()) + if(channel >= metrics[0].channel_count()) INTEROP_THROW(model::invalid_parameter, "Channel exceeds channel count"); for (const_iterator it = metrics.begin(); it != metrics.end(); ++it, ++focus_scores) *focus_scores = it->focus_score(channel); diff --git a/src/interop/logic/metric/q_metric.cpp b/src/interop/logic/metric/q_metric.cpp index f0ea7e994..98721e592 100644 --- a/src/interop/logic/metric/q_metric.cpp +++ b/src/interop/logic/metric/q_metric.cpp @@ -36,7 +36,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric const uint_t tile_id = *tile_beg; size_t prev_idx = metric_set.find(lane_id, tile_id, 1); if(prev_idx >= metric_set.size()) continue; - QMetric& metric = metric_set.at(prev_idx); + QMetric& metric = metric_set[prev_idx]; metric.accumulate(metric); const uint_t second_cycle_start = 2; // We have to accumulate the first cycle with itself, and every // subsequent with the previous cycle. @@ -46,7 +46,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric const size_t cur_idx = metric_set.find(lane_id, tile_id, cycle); if(cur_idx>=metric_set.size() || prev_idx>=metric_set.size()) continue;// TODO: if this happens zero out following q-scores - metric_set.at(cur_idx).accumulate(metric_set.at(prev_idx)); + metric_set[cur_idx].accumulate(metric_set[prev_idx]); prev_idx=cur_idx; } } @@ -70,7 +70,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric { size_t prev_idx = q_metric_set.find(*lane_beg, tile_id, 1); if(prev_idx >= q_metric_set.size()) continue; - q_by_lane_metric& metric = q_metric_set.at(prev_idx); + q_by_lane_metric& metric = q_metric_set[prev_idx]; metric.accumulate(metric); const uint_t second_cycle_start = 2; // We have to accumulate the first cycle with itself, and every // subsequent with the previous cycle. @@ -80,7 +80,7 @@ namespace illumina { namespace interop { namespace logic { namespace metric const size_t cur_idx = q_metric_set.find(*lane_beg, tile_id, cycle); if(cur_idx>=q_metric_set.size() || prev_idx>=q_metric_set.size()) continue;// TODO: if this happens zero out following q-scores - q_metric_set.at(cur_idx).accumulate(q_metric_set.at(prev_idx)); + q_metric_set[cur_idx].accumulate(q_metric_set[prev_idx]); prev_idx=cur_idx; } } diff --git a/src/tests/csharp/CMakeLists.txt b/src/tests/csharp/CMakeLists.txt index 626b104bf..0bdf77a4b 100644 --- a/src/tests/csharp/CMakeLists.txt +++ b/src/tests/csharp/CMakeLists.txt @@ -22,6 +22,7 @@ set(CSHARP_TEST_ENABLED ON PARENT_SCOPE) include(${CSHARP_USE_FILE}) set(TEST_SRCS + CoreTests.cs logic/ImagingTableLogic.cs logic/ExceptionTest.cs logic/IndexSummaryLogic.cs diff --git a/src/tests/csharp/CoreTests.cs b/src/tests/csharp/CoreTests.cs new file mode 100644 index 000000000..35f3de5f9 --- /dev/null +++ b/src/tests/csharp/CoreTests.cs @@ -0,0 +1,180 @@ +using System; +using NUnit.Framework; +using System.IO; +using Illumina.InterOp.Run; +using Illumina.InterOp.Metrics; +using Illumina.InterOp.Plot; +using Illumina.InterOp.Table; +using Illumina.InterOp.Comm; +using Illumina.InterOp.Summary; + +namespace Illumina.InterOp.Interop.UnitTest +{ + /// + /// Test C# Swig Wrapping + /// + [TestFixture] + public class CoreTests + { + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestHeatMapIndexOutOfBoundsException() + { + heatmap_data heatmap = new heatmap_data(); + System.Console.WriteLine(heatmap.at(0,0)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestImagingTableIndexOutOfBoundsException() + { + imaging_table table = new imaging_table(); + System.Console.WriteLine(table.at(0u,0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestIndexLaneSummaryIndexOutOfBoundsException() + { + index_lane_summary summary = new index_lane_summary(); + System.Console.WriteLine(summary.at(0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestIndexFlowcellSummaryIndexOutOfBoundsException() + { + index_flowcell_summary summary = new index_flowcell_summary(); + System.Console.WriteLine(summary.at(0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestRunSummaryIndexOutOfBoundsException() + { + run_summary summary = new run_summary(); + System.Console.WriteLine(summary.at(0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestReadSummaryIndexOutOfBoundsException() + { + read_summary summary = new read_summary(); + System.Console.WriteLine(summary.at(0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestLaneSummaryIndexOutOfBoundsException() + { + lane_summary summary = new lane_summary(); + System.Console.WriteLine(summary.at(0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestTileMetricsIndexOutOfBoundsException() + { + base_tile_metrics summary = new base_tile_metrics(); + System.Console.WriteLine(summary.at(0u)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("System.ArgumentOutOfRangeException")] + public void TestBarVectorIndexOutOfBoundsException() + { + bar_vector points = new bar_vector(); + System.Console.WriteLine(points[0]); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("System.ArgumentOutOfRangeException")] + public void TestCandleStickVectorIndexOutOfBoundsException() + { + candle_stick_vector points = new candle_stick_vector(); + System.Console.WriteLine(points[0]); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestCandleStickCollectionIndexOutOfBoundsException() + { + candle_stick_collection points = new candle_stick_collection(); + System.Console.WriteLine(points.at(0)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestBarCollectionIndexOutOfBoundsException() + { + bar_collection points = new bar_collection(); + System.Console.WriteLine(points.at(0)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestCandleStickSeriesIndexOutOfBoundsException() + { + candle_stick_series points = new candle_stick_series(); + System.Console.WriteLine(points.at(0)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestBarSeriesIndexOutOfBoundsException() + { + bar_series points = new bar_series(); + System.Console.WriteLine(points.at(0)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestCandleStickDataIndexOutOfBoundsException() + { + candle_stick_plot_data points = new candle_stick_plot_data(); + System.Console.WriteLine(points.at(0)); + } + /// + /// Test IndexOutOfBoundsException + /// + [Test] + [ExpectedException("Illumina.InterOp.Run.index_out_of_bounds_exception")] + public void TestBarDataIndexOutOfBoundsException() + { + bar_plot_data points = new bar_plot_data(); + System.Console.WriteLine(points.at(0)); + } + } +} diff --git a/src/tests/csharp/logic/ImagingTableLogic.cs b/src/tests/csharp/logic/ImagingTableLogic.cs index f7e0c0b4b..d958f3c35 100644 --- a/src/tests/csharp/logic/ImagingTableLogic.cs +++ b/src/tests/csharp/logic/ImagingTableLogic.cs @@ -98,7 +98,7 @@ public void PopulateTableTest() imaging_table table = new imaging_table(); c_csharp_table.create_imaging_table(run, table); Assert.AreEqual(table.row_count(), 3); - Assert.AreEqual(table.at(0, 0), 7); + Assert.AreEqual(table.at(0u, 0u), 7); } [Test] diff --git a/src/tests/interop/logic/index_summary_test.cpp b/src/tests/interop/logic/index_summary_test.cpp index 199246a5d..5296aa9e1 100644 --- a/src/tests/interop/logic/index_summary_test.cpp +++ b/src/tests/interop/logic/index_summary_test.cpp @@ -129,6 +129,27 @@ TEST(index_summary_test, lane_summary) } } +TEST(index_summary_test, lane_summary_cluster_count) +{ + const ::uint64_t cluster_count = static_cast< ::uint64_t >(std::numeric_limits< ::int64_t >::max() ); + model::metrics::run_metrics metrics; + std::vector< model::metrics::index_info > indices; + indices.reserve(2); + indices.push_back(model::metrics::index_info("TTGC", "Unknown", "Unknown", cluster_count)); + indices.push_back(model::metrics::index_info("AATG", "Unknown", "Unknown", cluster_count)); + + metrics.get().insert(model::metrics::index_metric(7, 1114, 1, indices)); + metrics.get().insert(model::metrics::index_metric(7, 1115, 1, indices)); + + metrics.get().insert(model::metrics::tile_metric(7, 1114, 10000, 10000, 10000, 10000)); + metrics.get().insert(model::metrics::tile_metric(7, 1115, 10000, 10000, 10000, 10000)); + + index_lane_summary summary; + logic::summary::summarize_index_metrics(metrics, 7, summary); + ASSERT_EQ(2u, summary.size()); + EXPECT_EQ(cluster_count*2u, summary[0].cluster_count()); +} + //--------------------------------------------------------------------------------------------------------------------- // Unit test section //--------------------------------------------------------------------------------------------------------------------- diff --git a/src/tests/interop/logic/plot_flowcell_test.cpp b/src/tests/interop/logic/plot_flowcell_test.cpp index 279de963c..8290b7bb6 100644 --- a/src/tests/interop/logic/plot_flowcell_test.cpp +++ b/src/tests/interop/logic/plot_flowcell_test.cpp @@ -45,7 +45,7 @@ TEST_P(flowcell_plot_tests, plot_data) ASSERT_EQ(actual.column_count(), expected.column_count()); for(size_t i=0, n=actual.length();i metrics; for(uint_t lane=1;lane<=8;++lane) { - for(uint_t surface=1;surface<=2;++surface) { - for (uint_t swath = 1; swath <= 4; ++swath) { - for (uint_t tile = 1; tile <= 36; ++tile) { + for(uint_t surface=1;surface<=2;++surface) + { + for (uint_t swath = 1; swath <= 4; ++swath) + { + for (uint_t tile = 1; tile <= 36; ++tile) + { tile_metric metric(lane, surface*1000+swath*100+tile, 0, 0, 0, 0); metrics.insert(metric.id(), metric); } @@ -150,8 +153,8 @@ TEST(tile_metrics_test, test_unique_id_four_digit) } for(size_t i=0;i metrics; for(uint_t lane=1;lane<=8;++lane) { - for(uint_t surface=1;surface<=2;++surface) { - for (uint_t swath = 1; swath <= 2; ++swath) { - for(uint_t section=1;section <=4;++section) { - for (uint_t tile = 1; tile <= 36; ++tile) { + for(uint_t surface=1;surface<=2;++surface) + { + for (uint_t swath = 1; swath <= 2; ++swath) + { + for(uint_t section=1;section <=4;++section) + { + for (uint_t tile = 1; tile <= 36; ++tile) + { tile_metric metric(lane, surface * 10000 + swath * 1000 + section*100 + tile, 0, 0, 0, 0); metrics.insert(metric); } @@ -176,8 +183,8 @@ TEST(tile_metrics_test, test_unique_id_five_digit) } for(size_t i=0;i percent_aligned_vec(expected.size()); - for (size_t i = 0; i < expected.size(); ++i) percent_aligned_vec[i] = expected.at(i).percent_aligned(read); + for (size_t i = 0; i < expected.size(); ++i) percent_aligned_vec[i] = expected[i].percent_aligned(read); float expected_percent_aligned_std = std::sqrt(interop::util::variance(percent_aligned_vec.begin(), percent_aligned_vec.end())); EXPECT_NEAR(expected_percent_aligned_std, 0.074578315019607544, tol); From b54c112ab4413bdf479ce97fefeae89efedb29e9 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Mon, 19 Dec 2016 16:56:45 -0800 Subject: [PATCH 8/8] Replace interop2csv with versioned dumptext --- docs/src/changes.md | 12 ++ interop/io/format/abstract_text_format.h | 72 ++++++++ interop/io/format/generic_layout.h | 7 + interop/io/format/metric_format_factory.h | 14 +- interop/io/format/text_format.h | 92 ++++++++++ interop/io/format/text_format_factory.h | 125 +++++++++++++ interop/io/metric_stream.h | 41 ++++- interop/model/metrics/extraction_metric.h | 53 +++++- interop/model/metrics/q_metric.h | 6 + interop/util/self_registration.h | 22 +++ src/apps/CMakeLists.txt | 1 + src/apps/dumptext.cpp | 164 ++++++++++++++++++ src/apps/interop2csv.cpp | 4 +- src/interop/CMakeLists.txt | 4 + .../metrics/corrected_intensity_metric.cpp | 88 +++++++++- src/interop/model/metrics/error_metric.cpp | 60 +++++++ .../model/metrics/extraction_metric.cpp | 80 +++++++++ src/interop/model/metrics/image_metric.cpp | 79 +++++++++ src/interop/model/metrics/index_metric.cpp | 66 +++++++ .../model/metrics/q_collapsed_metric.cpp | 62 +++++++ src/interop/model/metrics/q_metric.cpp | 102 +++++++++++ src/interop/model/metrics/tile_metric.cpp | 81 ++++++++- .../csharp/metrics/ExtractionMetricsTest.cs | 2 +- src/tests/csharp/metrics/PerformanceTest.cs | 2 +- 24 files changed, 1217 insertions(+), 22 deletions(-) create mode 100644 interop/io/format/abstract_text_format.h create mode 100644 interop/io/format/text_format.h create mode 100644 interop/io/format/text_format_factory.h create mode 100644 interop/util/self_registration.h create mode 100644 src/apps/dumptext.cpp diff --git a/docs/src/changes.md b/docs/src/changes.md index b352fcc1c..37efcd121 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -4,6 +4,18 @@ Date | Description ---------- | ----------- +2016-12-21 | Replace interop2csv with dumptext +2016-12-20 | IPA-5923: Fix bug in legacy q-metric binning +2016-12-19 | Removes the coverity run from the build matrix for every pull request +2016-12-16 | IPA-5885: Ensure error summary matches requirements +2016-12-14 | Enhance C# version information +2016-12-14 | Clean up unused enum types +2016-12-12 | IPA-5153: Ensure Summary tab calculations are consistent with Docs +2016-12-09 | IPA-5883: Fix possible memory issue in MSVC12 +2016-12-08 | IPA-4674: Add support for absolute naming convention +2016-12-07 | IPA-5869: Add section filtering +2016-12-06 | IPA-5734: Synchronize master +2016-12-02 | Add Google Analytics tracking to documentation 2016-12-08 | Added admonition to avoid using MSVC 12 (2013) with C# bindings 2016-12-08 | Added support for absolute tile naming 2016-12-08 | Added ability to do section filtering diff --git a/interop/io/format/abstract_text_format.h b/interop/io/format/abstract_text_format.h new file mode 100644 index 000000000..9a6c79610 --- /dev/null +++ b/interop/io/format/abstract_text_format.h @@ -0,0 +1,72 @@ +/** Metric format interface for the text format factory + * + * @file + * @date 12/19/16 + * @version 1.0 + * @copyright GNU Public License. + */ + +#pragma once + +#include +#include "interop/util/cstdint.h" + +namespace illumina { namespace interop { namespace io +{ + /** Abstract class that provides an interface for the text format of a metric + * + * The template argument for this class corresponds to a specific type + * of metric set. + */ + template + struct abstract_text_format + { + /** Define the metric type */ + typedef Metric metric_t; + /** Define the metric header type */ + typedef typename Metric::header_type header_t; + /** ID type */ + typedef typename metric_t::id_t id_t; + + /** Destructor + */ + virtual ~abstract_text_format() {} + /** Write the header for a set of metric records to the given output stream + * + * @param out output stream to write the binary InterOp file data + * @param header header of a metric set + * @param channel_names list of channel names + * @param sep column seperator + * @param eol row separator + * @return number of column headers + */ + virtual size_t write_header(std::ostream &out, + const header_t &header, + const std::vector& channel_names, + const char sep, + const char eol) = 0; + /** Write a metric record to the given output stream + * + * @param out output stream to write the binary InterOp file data + * @param metric interop metric data to write + * @param header interop metric header data to write + * @param sep column seperator + * @param eol row separator + * @param missing missing value indicator + * @return number of columns written + */ + virtual size_t write_metric(std::ostream &out, + const metric_t &metric, + const header_t &header, + const char sep, + const char eol, + const char missing) = 0; + + /** Get the version of this metric format + * + * @return version number + */ + virtual ::int16_t version() const=0; + }; +}}} + diff --git a/interop/io/format/generic_layout.h b/interop/io/format/generic_layout.h index 9fcf4bd67..cf7eb31ab 100644 --- a/interop/io/format/generic_layout.h +++ b/interop/io/format/generic_layout.h @@ -23,6 +23,13 @@ namespace illumina { namespace interop { namespace io template struct generic_layout; + /** Define a text layout of the metric + * + * This is currently only used for writing metrics + */ + template + struct text_layout; + }}} diff --git a/interop/io/format/metric_format_factory.h b/interop/io/format/metric_format_factory.h index f6f56532c..df3decf7a 100644 --- a/interop/io/format/metric_format_factory.h +++ b/interop/io/format/metric_format_factory.h @@ -10,6 +10,7 @@ #include #include "interop/util/map.h" #include "interop/util/assert.h" +#include "interop/util/self_registration.h" #include "interop/io/format/abstract_metric_format.h" #include "interop/util/unique_ptr.h" #include "interop/io/stream_exceptions.h" @@ -33,19 +34,6 @@ illumina::interop::io::metric_format_factory< Proxy > \ illumina_interop_io_##Type##Proxy##Version(new illumina::interop::io::metric_format >); -/** Ensure that static libraries are properly linked - * This must be used in a function that will definitely be linked. - * - * Tested for Microsoft Visual C++, GCC and CLang - */ -#define INTEROP_FORCE_LINK_USE(X) void force_link_metric_format(X*); force_link_metric_format(0); -/** Ensure that static libraries are properly linked - * This must be used in a file that may not be linked. - * - * Tested for Microsoft Visual C++, GCC and CLang - */ -#define INTEROP_FORCE_LINK_DEF(X) namespace illumina{namespace interop{namespace io{ void force_link_metric_format(X*){} }}} \ - void force_link_metric_format(X*){} // For Microsoft Visual C++ namespace illumina { namespace interop { namespace io { diff --git a/interop/io/format/text_format.h b/interop/io/format/text_format.h new file mode 100644 index 000000000..eeb488725 --- /dev/null +++ b/interop/io/format/text_format.h @@ -0,0 +1,92 @@ +/** Concrete text layout formats for the abstract text format interface. + * + * + * @file + * @date 12/19/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#ifdef _MSC_VER +#pragma warning(disable:4702) // MSVC warns that there is unreachable code +#endif + + +#include "interop/util/exception.h" +#include "interop/io/format/abstract_text_format.h" +#include "interop/io/format/generic_layout.h" + +namespace illumina { namespace interop { namespace io +{ + /** Shared functionality for writing text InterOp metrics + */ + template + struct text_format; + /** Shared functionality for writing text InterOp metrics + * + * Specialization for text_layout + */ + template + struct text_format > : public abstract_text_format + { + private: + typedef text_layout layout_t; + typedef typename Metric::id_t id_t; + public: + /** Define the metric type */ + typedef Metric metric_t; + /** Define the metric header type */ + typedef typename Metric::header_type header_t; + + /** Write the header of text format to the given output stream + * + * @param out output stream to write the binary InterOp file data + * @param header header of a metric set + * @param channel_names list of channel names + * @param sep column seperator + * @param eol row separator + * @return number of column headers + */ + size_t write_header(std::ostream &out, + const header_t &header, + const std::vector& channel_names, + const char sep, + const char eol) + { + out << "# " << Metric::prefix() << Metric::suffix() << sep; + out << Version << eol; + return layout_t::write_header(out, header, channel_names, sep, eol); + } + + /** Write a text record to the given output stream + * + * @param out output stream + * @param metric a metric to write + * @param header metric set header + * @param sep column seperator + * @param eol row separator + * @param missing missing value indicator + * @return number of columns written + */ + size_t write_metric(std::ostream &out, + const metric_t &metric, + const header_t &header, + const char sep, + const char eol, + const char missing) + { + return layout_t::write_metric(out, metric, header, sep, eol, missing); + } + + /** Get the version of this metric format + * + * @return version number + */ + ::int16_t version() const + { + return static_cast< ::int16_t >(Version); + } + }; +}}} + + diff --git a/interop/io/format/text_format_factory.h b/interop/io/format/text_format_factory.h new file mode 100644 index 000000000..a0246f10c --- /dev/null +++ b/interop/io/format/text_format_factory.h @@ -0,0 +1,125 @@ +/** Factory for generating text formats + * + * @file + * @date 12/19/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once + +#include +#include "interop/util/map.h" +#include "interop/util/assert.h" +#include "interop/util/self_registration.h" +#include "interop/io/format/abstract_text_format.h" +#include "interop/util/unique_ptr.h" +#include "interop/io/stream_exceptions.h" + +/** Register a text format with the factory + * + * @param Metric metric class + * @param Version version number + */ +#define INTEROP_REGISTER_METRIC_TEXT_LAYOUT(Metric, Version) \ + illumina::interop::io::text_format_factory_proxy< Metric > \ + illumina_interop_io_text_##Type##Metric##Version(new illumina::interop::io::text_format >); + + +namespace illumina { namespace interop { namespace io +{ + /** Factory for generating text formats + * + * This class defines static methods to register a metric format. The registered metric formats can + * be accessed through the `instance()` static function. + * + * @note this is not thread safe + */ + template + struct text_format_factory + { + /** Define the metric type */ + typedef Metric metric_type; + /** Define the abstract format type */ + typedef abstract_text_format abstract_text_format_t; + /** Define the header type */ + typedef typename Metric::header_type header_type; + /** Define a unique pointer to a metric format */ + typedef stdbp::unique_ptr metric_format_pointer; + /** Define a map between format version and the format */ + typedef INTEROP_UNORDERED_MAP(int, metric_format_pointer) text_format_map; + + /** Find a format for a given version + * + * If the format is not found, return null + * + * @param version version to search for + * @return pointer to format or null + */ + abstract_text_format_t* find(int version) + { + if(version < 0) version = m_latest_version; + typename text_format_map::iterator it = m_format_map.find(version); + if(it == m_format_map.end()) return 0; + return &(*it->second); + } + + /** Add a text format to the factory + * + * @param pformat format to add + */ + void add(abstract_text_format_t *pformat) + { + const int version = pformat->version(); + if(version > m_latest_version) m_latest_version = version; + m_format_map[version] = metric_format_pointer(pformat); + } + /** Get number of text formats + * + * @return number of text formats + */ + size_t size()const + { + return m_format_map.size(); + } + + /** Instance of the factory singleton + * + * @note this is not thread safe + * @return instance to singleton + */ + static text_format_factory &instance() + { + INTEROP_FORCE_LINK_USE(metric_type); + static text_format_factory _inst; + return _inst; + } + + private: + text_format_factory() : m_latest_version(-1) {} + text_format_map m_format_map; + int m_latest_version; + + }; + /** Proxy for registering text formats + */ + template + struct text_format_factory_proxy + { + /** Define the metric type */ + typedef Metric metric_type; + /** Define the abstract format type */ + typedef abstract_text_format abstract_text_format_t; + /** Constructor + * + * This constructor is used to statically register a text format in a source file. + */ + text_format_factory_proxy(abstract_text_format_t *pformat) + { + text_format_factory::instance().add(pformat); + } + }; + +}}} + + + diff --git a/interop/io/metric_stream.h b/interop/io/metric_stream.h index a0810c1e0..eb5305fef 100644 --- a/interop/io/metric_stream.h +++ b/interop/io/metric_stream.h @@ -11,6 +11,7 @@ #include "interop/util/exception.h" #include "interop/model/model_exceptions.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/paths.h" #include "interop/util/filesystem.h" #include "interop/util/assert.h" @@ -273,7 +274,7 @@ namespace illumina { namespace interop { namespace io format_map[version]->write_metric_header(out, header); } - /** Write a metric header to a binary Interop output stream + /** Write a set of metrics to a binary Interop output stream * * @param out output stream * @param metrics set of metrics @@ -300,6 +301,44 @@ namespace illumina { namespace interop { namespace io it != metrics.end(); it++) format_map[version]->write_metric(out, *it, metrics); } + /** Write a set of metrics to a text output stream + * + * @param out output stream + * @param metrics set of metrics + * @param channel_names list of channel names + * @param version version of the InterOp to write (if less than 0, get from metric set) + * @param sep column separator + * @param eol row separator + * @param missing missing value indicator + */ + template + static void write_text(std::ostream &out, + const MetricSet &metrics, + const std::vector& channel_names, + ::int16_t version = -1, + const char sep=',', + const char eol='\n', + const char missing='-') + { + typedef typename MetricSet::metric_type metric_type; + typedef text_format_factory factory_type; + typedef typename factory_type::abstract_text_format_t* abstract_text_format_pointer_t; + + factory_type &factory = factory_type::instance(); + abstract_text_format_pointer_t format = factory.find(version); + if (format == 0) + INTEROP_THROW(bad_format_exception, + "No format found to write file with version: " + << version << " of " << factory.size() + << " for " << metric_type::prefix() << "" << metric_type::suffix() + << " with " << metrics.size() << " metrics"); + INTEROP_ASSERT(format); + format->write_header(out, metrics, channel_names, sep, eol); + for (typename MetricSet::const_iterator it = metrics.begin(); + it != metrics.end(); it++) + format->write_metric(out, *it, metrics, sep, eol, missing); + + } }}} diff --git a/interop/model/metrics/extraction_metric.h b/interop/model/metrics/extraction_metric.h index a34df1251..80edea5d9 100644 --- a/interop/model/metrics/extraction_metric.h +++ b/interop/model/metrics/extraction_metric.h @@ -25,6 +25,49 @@ namespace illumina { namespace interop { namespace model { namespace metrics { + /** Header information for writing an image metric set + */ + class extraction_metric_header : public metric_base::base_cycle_metric::header_type + { + public: + enum{ + /** Maximum number of channels supported **/ + MAX_CHANNELS=4 + }; + /** Unsigned int16_t + */ + typedef ::uint16_t ushort_t; + public: + /** Constructor + * + * @param channel_count number of channels + */ + extraction_metric_header(ushort_t channel_count) : m_channel_count(channel_count) {} + /** Number of channels + * + * @return number of channels + */ + ushort_t channel_count()const{return m_channel_count;} + /** Generate a default header + * + * @return default header + */ + static extraction_metric_header default_header() + { + return extraction_metric_header(MAX_CHANNELS); + } + /** Clear the data + */ + void clear() + { + m_channel_count=MAX_CHANNELS; + metric_base::base_cycle_metric::header_type::clear(); + } + private: + ushort_t m_channel_count; + template + friend struct io::generic_layout; + }; /** Extraction metric * * The extraction metrics include the max intensity and the focus score for each color channel. @@ -43,6 +86,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** Latest version of the InterOp format */ LATEST_VERSION = 2 }; + /** Extraction metric header */ + typedef extraction_metric_header header_type; /** Define a uint16_t array using an underlying vector */ typedef std::vector ushort_array_t; @@ -68,13 +113,15 @@ namespace illumina { namespace interop { namespace model { namespace metrics { } /** Constructor + * + * @param header extraction metric header */ - extraction_metric(const header_type&) : + extraction_metric(const header_type& header) : metric_base::base_cycle_metric(0, 0, 0), m_date_time_csharp(0), m_date_time(0), - m_max_intensity_values(MAX_CHANNELS, 0), - m_focus_scores(MAX_CHANNELS, 0) + m_max_intensity_values(header.channel_count(), 0), + m_focus_scores(header.channel_count(), 0) { } diff --git a/interop/model/metrics/q_metric.h b/interop/model/metrics/q_metric.h index cde388be9..9a13a1985 100644 --- a/interop/model/metrics/q_metric.h +++ b/interop/model/metrics/q_metric.h @@ -171,6 +171,12 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ size_t bin_count() const { return m_qscore_bins.size(); } + /** Get the number of bins in header + * + * @return number of bins in header + */ + size_t q_val_count() const + { return m_qscore_bins.empty() ? static_cast(MAX_Q_BINS) : m_qscore_bins.size(); } /** Get the index for the given q-value * diff --git a/interop/util/self_registration.h b/interop/util/self_registration.h new file mode 100644 index 000000000..f824c065e --- /dev/null +++ b/interop/util/self_registration.h @@ -0,0 +1,22 @@ +/** Ensure self registration classes properly link + * + * @file + * @date 12/19/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once + +/** Ensure that static libraries are properly linked + * This must be used in a function that will definitely be linked. + * + * Tested for Microsoft Visual C++, GCC and CLang + */ +#define INTEROP_FORCE_LINK_USE(X) void force_link_metric_format(X*); force_link_metric_format(0); +/** Ensure that static libraries are properly linked + * This must be used in a file that may not be linked. + * + * Tested for Microsoft Visual C++, GCC and CLang + */ +#define INTEROP_FORCE_LINK_DEF(X) namespace illumina{namespace interop{namespace io{ void force_link_metric_format(X*){} }}} \ + void force_link_metric_format(X*){} // For Microsoft Visual C++ diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt index 7e060e754..28aacfeb2 100644 --- a/src/apps/CMakeLists.txt +++ b/src/apps/CMakeLists.txt @@ -31,6 +31,7 @@ endfunction() set(SWIG_VERSION_INFO "") add_application(interop2csv interop2csv.cpp) +add_application(dumptext dumptext.cpp) add_application(cyclesim cyclesim.cpp) add_application(summary summary.cpp) add_application(index-summary index_summary.cpp) diff --git a/src/apps/dumptext.cpp b/src/apps/dumptext.cpp new file mode 100644 index 000000000..fd16f6da8 --- /dev/null +++ b/src/apps/dumptext.cpp @@ -0,0 +1,164 @@ +/** @page dumpbin Dump Interop data into a text format + * + * + * This developer application is to help debug InterOp files. + * + * ### Running the Program + * + * The program runs as follows: + * + * $ dumptext 140131_1287_0851_A01n401drr + * + * In this sample, 140131_1287_0851_A01n401drr is a run folder and the summary is written to the standard output. + * + * # Version: v1.0.4-70-g9bcfb5a-dirty + * + */ + +#include +#include +#include "interop/io/metric_file_stream.h" +#include "interop/model/run_metrics.h" +#include "interop/util/option_parser.h" +#include "interop/version.h" +#include "inc/application.h" + +using namespace illumina::interop::model::metrics; +using namespace illumina::interop; + +/** Call back functor for writing text data to the output stream + */ +struct metric_writer +{ + /** Constructor + * + * @param out output stream + * @param channels list of channel names + */ + metric_writer(std::ostream& out, const std::vector& channels) : m_out(out), m_channel_names(channels){} + /** Function operator overload to write data + * + * @param metrics set of metrics + */ + template + void operator()(const MetricSet& metrics)const + { + if(metrics.empty()) return; + io::write_text(m_out, metrics, m_channel_names); + } +private: + std::ostream& m_out; + std::vector m_channel_names; + +}; +/** Copy of subset of metrics + */ +struct subset_copier +{ + /** Constructor + * + * @param run run metrics + * @param total number to subsample + */ + subset_copier(run_metrics& run, const size_t total) : m_run(run), m_total(total){} + + /** Function operator overload to collect a subset of metrics + * + * @param metrics set of metrics + */ + template + void operator()(const MetricSet& metrics)const + { + m_run.get() = MetricSet(metrics, metrics.version()); + const size_t total = std::min(m_total, metrics.size()); + + for(size_t i=0;i().insert(metrics.at(i)); + } +private: + run_metrics& m_run; + size_t m_total; +}; + +int main(int argc, const char** argv) +{ + if(argc == 0) + { + std::cerr << "No arguments specified!" << std::endl; + //print_help(std::cout); + return INVALID_ARGUMENTS; + } + + const char eol = '\n'; + + std::cout << "# Version: " << INTEROP_VERSION << std::endl; + + size_t subset_count=0; + util::option_parser description; + description + (subset_count, "subset", "Number of metrics to subsample"); + if(description.is_help_requested(argc, argv)) + { + std::cout << "Usage: " << io::basename(argv[0]) << " run_folder [--option1=value1] [--option2=value2]" << std::endl; + description.display_help(std::cout); + return SUCCESS; + } + try{ + description.parse(argc, argv); + description.check_for_unknown_options(argc, argv); + } + catch(const util::option_exception& ex) + { + std::cerr << ex.what() << std::endl; + return INVALID_ARGUMENTS; + } + + for(int i=1;i 0 ) + { + run_metrics subset; + subset_copier copy_subset(subset, subset_count); + try + { + run.metrics_callback(copy_subset); + } + catch (const std::exception &ex) + { + std::cerr << ex.what() << std::endl; + return UNEXPECTED_EXCEPTION; + } + try + { + subset.metrics_callback(write_metrics); + } + catch (const io::bad_format_exception &ex) + { + std::cerr << ex.what() << std::endl; + return BAD_FORMAT; + } + } + else + { + try + { + run.metrics_callback(write_metrics); + } + catch (const io::bad_format_exception &ex) + { + std::cerr << ex.what() << std::endl; + return BAD_FORMAT; + } + } + std::cout << eol << std::endl; + + } + return SUCCESS; +} + + + diff --git a/src/apps/interop2csv.cpp b/src/apps/interop2csv.cpp index 8b24e77de..144e7c3cd 100644 --- a/src/apps/interop2csv.cpp +++ b/src/apps/interop2csv.cpp @@ -4,6 +4,8 @@ * the data as a special type of CSV (common separated format) file. The program takes the run folder as an input * and then writes the data as plain text to the console. This data can then be redirected to a file. * + * @warning This program is deprecated and will be removed. Use dumptext instead + * * ### Running the Program * * The program runs as follows: @@ -694,7 +696,7 @@ int write_index_metrics(std::ostream& out, const std::string& filename) if(res != 0) return res; write_header(out, metrics); - out << "Lane,Tile,Read,Sequence,Sample,Project,Count\n"; + out << "Lane,Tile,Read,Sequence,Sample,Project,ClusterCount\n"; for(index_metric_set::metric_array_t::const_iterator beg = metrics.begin(), end = vec_end(metrics);beg != end;++beg) { const index_metric& metric = *beg; diff --git a/src/interop/CMakeLists.txt b/src/interop/CMakeLists.txt index d9d9a6df0..ec20e6b82 100644 --- a/src/interop/CMakeLists.txt +++ b/src/interop/CMakeLists.txt @@ -150,6 +150,10 @@ set(HEADERS ../../interop/util/map.h ../../interop/util/timer.h ../../interop/constants/enum_description.h + ../../interop/io/format/abstract_text_format.h + ../../interop/io/format/text_format.h + ../../interop/util/self_registration.h + ../../interop/io/format/text_format_factory.h ) set(INTEROP_HEADERS ${HEADERS} PARENT_SCOPE) diff --git a/src/interop/model/metrics/corrected_intensity_metric.cpp b/src/interop/model/metrics/corrected_intensity_metric.cpp index bf0903e86..1052ac04f 100644 --- a/src/interop/model/metrics/corrected_intensity_metric.cpp +++ b/src/interop/model/metrics/corrected_intensity_metric.cpp @@ -10,8 +10,11 @@ #include "interop/model/metrics/corrected_intensity_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" +#include "interop/io/format/text_format.h" +#include "interop/logic/utils/enums.h" using namespace illumina::interop::model::metrics; @@ -219,10 +222,93 @@ namespace illumina{ namespace interop{ namespace io }; #pragma pack() + + + /** Corrected intensity Metric CSV text format + * + * This class provide an interface for writing the corrected intensity metrics to a CSV file: + * + * - CorrectedIntMetrics.csv + */ + template<> + struct text_layout< corrected_intensity_metric, 1 > + { + /** Define a header type */ + typedef corrected_intensity_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type&, + const std::vector&, + const char sep, + const char eol) + { + const char* column_headers[] = + { + "Lane", "Tile", "Cycle", "AverageCycleIntensity", "SignalToNoise" + }; + const char* subcolumn_headers[] = + { + "CalledCount", "CalledIntensity", "AllIntensity" + }; + std::vector bases; + constants::list_enum_names(bases); + std::vector headers; + headers.reserve(util::length_of(column_headers)+util::length_of(subcolumn_headers)*5); + for(size_t i=0;i(constants::NUM_OF_BASES_AND_NC);++i) + headers.push_back(std::string()+subcolumn_headers[0]+"_"+bases[i]); + for(size_t i=1;i(constants::NUM_OF_BASES_AND_NC);++j) + headers.push_back(std::string()+subcolumn_headers[i]+"_"+bases[j]); + } + out << "# Column Count: " << util::length_of(headers) << eol; + out << headers[0]; + for(size_t i=1;i(i)); + for(size_t i=0;i(i)); + for(size_t i=0;i(i)); + out << eol; + return 0; + } + }; }}} +INTEROP_FORCE_LINK_DEF(corrected_intensity_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(corrected_intensity_metric, 2 ) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(corrected_intensity_metric, 3 ) -INTEROP_FORCE_LINK_DEF(corrected_intensity_metric) +// Text formats +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(corrected_intensity_metric, 1) diff --git a/src/interop/model/metrics/error_metric.cpp b/src/interop/model/metrics/error_metric.cpp index 0a1a3250a..e63a424b4 100644 --- a/src/interop/model/metrics/error_metric.cpp +++ b/src/interop/model/metrics/error_metric.cpp @@ -9,8 +9,10 @@ */ #include "interop/model/metrics/error_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" +#include "interop/io/format/text_format.h" using namespace illumina::interop::model::metrics; @@ -110,8 +112,66 @@ namespace illumina{ namespace interop{ namespace io #pragma pack() + /** Error Metric CSV text format + * + * This class provide an interface for writing the error metrics to a CSV file: + * + * - ErrorMetrics.csv + */ + template<> + struct text_layout< error_metric, 1 > + { + /** Define a header type */ + typedef error_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type&, + const std::vector&, + const char sep, + const char eol) + { + const char* headers[] = + { + "Lane", "Tile", "Cycle", "ErrorRate" + }; + out << "# Column Count: " << util::length_of(headers) << eol; + out << headers[0]; + for(size_t i=1;i + struct text_layout< extraction_metric, 1 > + { + /** Define a header type */ + typedef extraction_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param header extraction metric header + * @param channel_names list of channel names + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type& header, + const std::vector& channel_names, + const char sep, + const char eol) + { + if( static_cast(header.channel_count()) != channel_names.size() ) + INTEROP_THROW(bad_format_exception, "Header and channel names count mismatch"); + const char* headers[] = + { + "Lane", "Tile", "Cycle", "TimeStamp" + }; + out << "# Column Count: " << util::length_of(headers)+header.channel_count()*2 << eol; + out << "# Channel Count: " << header.channel_count() << eol; + out << headers[0]; + for(size_t i=1;i(header.channel_count());++i) + out << sep << max_intensity << "_" << channel_names[i]; + const std::string focus = "Focus"; + for(size_t i=0;i(header.channel_count());++i) + out << sep << focus << "_" << channel_names[i]; + out << eol; + return util::length_of(headers); + } + /** Write a extraction metric to the output stream + * + * @param out output stream + * @param metric extraction metric + * @param header extraction metric header + * @param sep column separator + * @param eol row separator + * @return number of columns written + */ + static size_t write_metric(std::ostream& out, + const extraction_metric& metric, + const header_type& header, + const char sep, + const char eol, + const char) + { + if( header.channel_count() != metric.channel_count() ) + INTEROP_THROW(bad_format_exception, "Header and metric channel count mismatch"); + out << metric.lane() << sep << metric.tile() << sep << metric.cycle() << sep; + out << metric.date_time(); // TODO: Format date/time + for(size_t i=0;i(header.channel_count());i++) + out << sep << metric.max_intensity(i); + for(size_t i=0;i(header.channel_count());i++) + out << sep << metric.focus_score(i); + out << eol; + return 0; + } + }; }}} INTEROP_FORCE_LINK_DEF(extraction_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(extraction_metric, 2) + +// Text formats +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(extraction_metric, 1) + diff --git a/src/interop/model/metrics/image_metric.cpp b/src/interop/model/metrics/image_metric.cpp index 4a452588c..424656a6a 100644 --- a/src/interop/model/metrics/image_metric.cpp +++ b/src/interop/model/metrics/image_metric.cpp @@ -11,8 +11,10 @@ #include #include "interop/model/metrics/image_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" +#include "interop/io/format/text_format.h" using namespace illumina::interop::model::metrics; @@ -264,6 +266,79 @@ namespace illumina { namespace interop { namespace io #pragma pack() // DO NOT MOVE + /** Image Metric CSV text format + * + * This class provide an interface for writing the image metrics to a CSV file: + * + * - ImageMetrics.csv + */ + template<> + struct text_layout< image_metric, 1 > + { + /** Define a header type */ + typedef image_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param header image metric header + * @param channel_names list of channel names + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type& header, + const std::vector& channel_names, + const char sep, + const char eol) + { + if( static_cast(header.channel_count()) != channel_names.size() ) + INTEROP_THROW(bad_format_exception, "Header and channel names count mismatch"); + const char* headers[] = + { + "Lane", "Tile", "Cycle" + }; + out << "# Column Count: " << util::length_of(headers)+header.channel_count()*2 << eol; + out << "# Channel Count: " << header.channel_count() << eol; + out << headers[0]; + for(size_t i=1;i(header.channel_count());++i) + out << sep << min_contrast << "_" << channel_names[i]; + const std::string max_contrast = "MaxContrast"; + for(size_t i=0;i(header.channel_count());++i) + out << sep << max_contrast << "_" << channel_names[i]; + out << eol; + return util::length_of(headers); + } + /** Write a image metric to the output stream + * + * @param out output stream + * @param metric image metric + * @param header image metric header + * @param sep column separator + * @param eol row separator + * @return number of columns written + */ + static size_t write_metric(std::ostream& out, + const image_metric& metric, + const header_type& header, + const char sep, + const char eol, + const char) + { + if( header.channel_count() != metric.channel_count() ) + INTEROP_THROW(bad_format_exception, "Header and metric channel count mismatch"); + out << metric.lane() << sep << metric.tile() << sep << metric.cycle() << sep; + for(size_t i=0;i(header.channel_count());i++) + out << sep << metric.min_contrast(i); + for(size_t i=0;i(header.channel_count());i++) + out << sep << metric.max_contrast(i); + out << eol; + return 0; + } + }; }}} INTEROP_FORCE_LINK_DEF(image_metric) @@ -271,3 +346,7 @@ INTEROP_FORCE_LINK_DEF(image_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 1) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(image_metric, 2) + +// Text formats +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(image_metric, 1) + diff --git a/src/interop/model/metrics/index_metric.cpp b/src/interop/model/metrics/index_metric.cpp index 659fc1441..062889dba 100644 --- a/src/interop/model/metrics/index_metric.cpp +++ b/src/interop/model/metrics/index_metric.cpp @@ -10,8 +10,10 @@ #include #include "interop/model/metrics/index_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" +#include "interop/io/format/text_format.h" using namespace illumina::interop::model::metrics; @@ -172,9 +174,73 @@ namespace illumina { namespace interop { namespace io }; #pragma pack() + /** Tile Metric CSV text format + * + * This class provide an interface for writing the tile metrics to a CSV file: + * + * - TileMetrics.csv + */ + template<> + struct text_layout< index_metric, 1 > + { + /** Define a header type */ + typedef index_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type&, + const std::vector&, + const char sep, + const char eol) + { + const char* headers[] = + { + "Lane", "Tile", "Read", "Sequence", "Sample", "Project", "ClusterCount" + }; + out << "# Column Count: " << util::length_of(headers) << eol; + out << headers[0]; + for(size_t i=1;iindex_seq() << sep << index_beg->sample_id(); + out << sep << index_beg->sample_proj() << sep << index_beg->cluster_count() << eol; + } + return 0; + } + }; }}} INTEROP_FORCE_LINK_DEF(index_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(index_metric, 1) + +// Text formats +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(index_metric, 1) + diff --git a/src/interop/model/metrics/q_collapsed_metric.cpp b/src/interop/model/metrics/q_collapsed_metric.cpp index 07e87f80a..72bee1904 100644 --- a/src/interop/model/metrics/q_collapsed_metric.cpp +++ b/src/interop/model/metrics/q_collapsed_metric.cpp @@ -13,8 +13,10 @@ #include "interop/util/assert.h" #include "interop/model/metrics/q_collapsed_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" +#include "interop/io/format/text_format.h" using namespace illumina::interop::model::metrics; @@ -567,6 +569,62 @@ namespace illumina{ namespace interop{ namespace io { */ enum{/**Version of the format */VERSION=6}; }; + /** Tile Metric CSV text format + * + * This class provide an interface for writing the tile metrics to a CSV file: + * + * - TileMetrics.csv + */ + template<> + struct text_layout< q_collapsed_metric, 1 > + { + /** Define a header type */ + typedef q_collapsed_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type&, + const std::vector&, + const char sep, + const char eol) + { + const char* headers[] = + { + "Lane", "Tile", "Cycle", + "Q20", "Q30", "Total", "MedianQScore" + }; + out << "# Column Count: " << util::length_of(headers) << eol; + out << headers[0]; + for(size_t i=1;i + struct text_layout< q_metric, 1 > + { + /** Define a header type */ + typedef q_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param header q-metric header + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type& header, + const std::vector&, + const char sep, + const char eol) + { + const char* headers[] = + { + "Lane", "Tile", "Cycle" + }; + out << "# Bin Count: " << header.q_val_count() << eol; + if(header.bin_count() > 0) + { + const char* bin_headers[] = + { + "Lower", "Value", "Upper" + }; + out << "# Column Count: " << util::length_of(bin_headers) << eol; + out << bin_headers[0]; + for(size_t i=1;i(header.q_val_count()) != metric.size() ) + INTEROP_THROW(bad_format_exception, "Header and metric bin count mismatch: " + << header.q_val_count() << " != " << metric.size()); + out << metric.lane() << sep << metric.tile() << sep << metric.cycle(); + for(size_t i=0;i + struct text_layout< q_by_lane_metric, 1 > : public text_layout< q_metric, 1 > + { + }; }}} INTEROP_FORCE_LINK_DEF(q_metric) @@ -753,3 +852,6 @@ INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_by_lane_metric, 4) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_by_lane_metric, 5) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(q_by_lane_metric, 6) +// Text formats +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(q_metric, 1) +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(q_by_lane_metric, 1) diff --git a/src/interop/model/metrics/tile_metric.cpp b/src/interop/model/metrics/tile_metric.cpp index a4952dec8..4561020c7 100644 --- a/src/interop/model/metrics/tile_metric.cpp +++ b/src/interop/model/metrics/tile_metric.cpp @@ -11,14 +11,15 @@ #include "interop/util/math.h" #include "interop/model/metrics/tile_metric.h" #include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" #include "interop/io/format/metric_format.h" +#include "interop/io/format/text_format.h" using namespace illumina::interop::model::metrics; namespace illumina { namespace interop { namespace io { - #pragma pack(1) /** Tile Metric Record Layout Version 2 @@ -266,8 +267,86 @@ namespace illumina { namespace interop { namespace io #pragma pack() + /** Tile Metric CSV text format + * + * This class provide an interface for writing the tile metrics to a CSV file: + * + * - TileMetrics.csv + */ + template<> + struct text_layout< tile_metric, 1 > + { + /** Define a header type */ + typedef tile_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type&, + const std::vector&, + const char sep, + const char eol) + { + const char* headers[] = + { + "Lane", "Tile", "Read", + "ClusterCount", "ClusterCountPF", "Density", "DensityPF", "Aligned", "Prephasing", "Phasing" + }; + out << "# Column Count: " << util::length_of(headers) << eol; + out << headers[0]; + for(size_t i=1;iread() << sep; + out << metric.cluster_count() << sep << metric.cluster_count_pf() << sep; + out << metric.cluster_density() << sep << metric.cluster_density_pf() << sep; + out << rbeg->percent_aligned() << sep << rbeg->percent_prephasing() + << sep << rbeg->percent_phasing(); + out << eol; + } + } + return 0; + } + }; }}} INTEROP_FORCE_LINK_DEF(tile_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(tile_metric, 2) +// Text formats +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(tile_metric, 1) diff --git a/src/tests/csharp/metrics/ExtractionMetricsTest.cs b/src/tests/csharp/metrics/ExtractionMetricsTest.cs index 623f205bb..e00960045 100644 --- a/src/tests/csharp/metrics/ExtractionMetricsTest.cs +++ b/src/tests/csharp/metrics/ExtractionMetricsTest.cs @@ -24,7 +24,7 @@ public class ExtractionMetricsTestV2 [SetUp] protected void SetUp() { - base_cycle_metric_header header = new base_cycle_metric_header(); + extraction_metric_header header = new extraction_metric_header(2); float[] focus1 = new float[]{2.24664021f, 2.1896739f, 0, 0}; ushort[] p90_1 = new ushort[]{302, 273, 0, 0}; expected_metrics.Add(new extraction_metric(7, 1114, 1, new csharp_date_time(9859129975844165472ul), (p90_1), (focus1), 4)); diff --git a/src/tests/csharp/metrics/PerformanceTest.cs b/src/tests/csharp/metrics/PerformanceTest.cs index 30be2e6ca..24e62e9a2 100644 --- a/src/tests/csharp/metrics/PerformanceTest.cs +++ b/src/tests/csharp/metrics/PerformanceTest.cs @@ -21,7 +21,7 @@ public class ExtractionMetricsPerformanceTestV2 [SetUp] protected void SetUp() { - base_cycle_metric_header header = new base_cycle_metric_header(); + extraction_metric_header header = new extraction_metric_header(2); if(metrics.Count == 0) { System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();