From 73c3c2aa6ffb29f7b6d2fc80984aee20790b0671 Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Fri, 13 Nov 2020 06:26:16 -0800 Subject: [PATCH] Issue-?: Support % loading concentration (#236) Bump tests Bump tests Fix another warning Fix warnings Fix Python unit test Fix more Fix IUO issues --- docs/src/binary_formats.md | 1 + docs/src/changes.md | 10 +- docs/src/index.md | 2 + interop/constants/enums.h | 24 +- interop/constants/typedefs.h | 2 + interop/io/format/metric_format.h | 1 - interop/io/layout/base_metric.h | 47 ++++ interop/io/metric_file_stream.h | 6 +- interop/logic/summary/summary_statistics.h | 26 +++ interop/logic/summary/tile_summary.h | 14 +- interop/model/metric_base/base_metric.h | 55 ++++- interop/model/metric_base/metric_set.h | 71 +++++- interop/model/metrics/summary_run_metric.h | 212 ++++++++++++++++++ interop/model/run_metrics.h | 9 +- interop/model/summary/metric_summary.h | 24 +- interop/model/summary/stat_summary.h | 2 - interop/util/exception_specification.h | 2 +- interop/util/type_traits.h | 5 +- src/apps/aggregate.cpp | 2 + src/ext/swig/arrays/arrays_numpy_impl.i | 4 +- src/ext/swig/metrics.i | 1 + src/interop/CMakeLists.txt | 5 +- src/interop/logic/summary/run_summary.cpp | 18 ++ .../model/metrics/summary_run_metric.cpp | 192 ++++++++++++++++ src/interop/model/run_metrics.cpp | 10 +- src/interop/model/run_metrics_helper.cpp | 2 + src/tests/interop/CMakeLists.txt | 2 + .../metrics/extended_tile_metrics_test.cpp | 2 - .../metrics/inc/metric_format_fixtures.h | 4 +- .../interop/metrics/inc/metric_generator.h | 5 +- .../metrics/inc/summary_run_metrics_test.h | 52 +++++ .../interop/metrics/metric_streams_test.cpp | 1 + src/tests/interop/metrics/run_metric_test.cpp | 12 +- .../metrics/summary_run_metrics_test.cpp | 74 ++++++ src/tests/python/CoreTests.py | 32 ++- 35 files changed, 875 insertions(+), 56 deletions(-) create mode 100644 interop/model/metrics/summary_run_metric.h create mode 100644 src/interop/model/metrics/summary_run_metric.cpp create mode 100644 src/tests/interop/metrics/inc/summary_run_metrics_test.h create mode 100644 src/tests/interop/metrics/summary_run_metrics_test.cpp diff --git a/docs/src/binary_formats.md b/docs/src/binary_formats.md index b1ad8a49f..bf5fee552 100644 --- a/docs/src/binary_formats.md +++ b/docs/src/binary_formats.md @@ -44,6 +44,7 @@ The documentation for the model notes when an attribute is only populated by a s - @subpage q_collapsed_v4 "Collapsed Q-Metrics Version 4" - @subpage q_collapsed_v5 "Collapsed Q-Metrics Version 5" - @subpage q_collapsed_v6 "Collapsed Q-Metrics Version 6" + - @subpage summary_run_v1 "Summary Run Version 1" The following are binary formats used only for testing purposes and are not officially supported: diff --git a/docs/src/changes.md b/docs/src/changes.md index 5466096c9..65964ea03 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -1,5 +1,13 @@ # Changes {#changes} - + +## v1.1.15 + +Date | Description +---------- | ----------- +2020-08-26 | Issue-229: Support % loading concentration +2020-08-26 | Issue-229: Fix Python binding for read_metrics_from_buffer + + ## v1.1.14 Date | Description diff --git a/docs/src/index.md b/docs/src/index.md index c1f6b8732..0f6d7c158 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -58,6 +58,7 @@ List of InterOp Metric Files | [QMetricsByLaneOut.bin] | Per tile per cycle Q-score histogram per lane | | [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle | | [ExtendedTileMetricsOut.bin] | Per tile occupancy metrics | +| [SummaryRunMetricsOut.bin] | Per run summary metrics | [CorrectedIntMetricsOut.bin]: @ref corrected_intensity "CorrectedIntMetricsOut.bin" [ErrorMetricsOut.bin]: @ref error_metric "ErrorMetricsOut.bin" @@ -70,6 +71,7 @@ List of InterOp Metric Files [QMetricsByLaneOut.bin]: @ref q_metric_by_lane "QMetricsByLaneOut.bin" [EmpiricalPhasingMetricsOut.bin]: @ref phasing_metric "EmpiricalPhasingMetricOut.bin" [ExtendedTileMetricsOut.bin]: @ref extended_tile_metric "ExtendedTileMetricsOut.bin" +[SummaryRunMetricsOut.bin]: @ref summary_run_metric "SummaryRunMetricsOut.bin" Known Limitations diff --git a/interop/constants/enums.h b/interop/constants/enums.h index 097ad901a..4a1a2d787 100644 --- a/interop/constants/enums.h +++ b/interop/constants/enums.h @@ -14,7 +14,7 @@ #include "interop/util/cstdint.h" /** Sentinel for an unknown enum type */ -#define INTEROP_UNKNOWN 0x400 +#define INTEROP_UNKNOWN 0x800 /** Enumeration of specific features that can belong to a metric @@ -30,6 +30,7 @@ INTEROP_TUPLE_ASSIGN(BaseFeature, 0x08), \ INTEROP_TUPLE_ASSIGN(ChannelFeature, 0x10), \ INTEROP_TUPLE_ASSIGN(LaneFeature, 0x20), \ + INTEROP_TUPLE_ASSIGN(DiskFeature, 0x400), \ INTEROP_TUPLE1(UnknownMetricFeature) //NOTE: if we add any more features above, we should update the unknown metric value below @@ -74,18 +75,19 @@ * @see illumina::interop::constants::metric_group */ #define INTEROP_ENUM_METRIC_GROUPS \ - INTEROP_TUPLE2(CorrectedInt, CycleFeature|BaseFeature), \ - INTEROP_TUPLE2(Error, CycleFeature), \ - INTEROP_TUPLE2(Extraction, CycleFeature|ChannelFeature), \ - INTEROP_TUPLE2(Image, CycleFeature|ChannelFeature), \ - INTEROP_TUPLE2(Index, ReadFeature), \ - INTEROP_TUPLE2(Q, CycleFeature), \ - INTEROP_TUPLE2(Tile, TileFeature), \ + INTEROP_TUPLE2(CorrectedInt, CycleFeature|BaseFeature|DiskFeature), \ + INTEROP_TUPLE2(Error, CycleFeature|DiskFeature), \ + INTEROP_TUPLE2(Extraction, CycleFeature|ChannelFeature|DiskFeature), \ + INTEROP_TUPLE2(Image, CycleFeature|ChannelFeature|DiskFeature), \ + INTEROP_TUPLE2(Index, ReadFeature|DiskFeature), \ + INTEROP_TUPLE2(Q, CycleFeature|DiskFeature), \ + INTEROP_TUPLE2(Tile, TileFeature|DiskFeature), \ INTEROP_TUPLE2(QByLane, LaneFeature), \ INTEROP_TUPLE2(QCollapsed, CycleFeature), \ - INTEROP_TUPLE2(EmpiricalPhasing, CycleFeature), \ + INTEROP_TUPLE2(EmpiricalPhasing, CycleFeature|DiskFeature), \ INTEROP_TUPLE2(DynamicPhasing, CycleFeature), \ - INTEROP_TUPLE2(ExtendedTile, TileFeature), \ + INTEROP_TUPLE2(ExtendedTile, TileFeature|DiskFeature), \ + INTEROP_TUPLE2(SummaryRun, DiskFeature), \ INTEROP_TUPLE1(MetricCount),\ INTEROP_TUPLE1(UnknownMetricGroup) @@ -187,6 +189,8 @@ INTEROP_TUPLE1(BaseReadType),\ /** Lane base types are written out once for each lane and cycle */\ INTEROP_TUPLE1(BaseLaneType),\ + /** Run base types are written out once per run */\ + INTEROP_TUPLE1(BaseRunType), \ INTEROP_TUPLE1(BaseMetricCount),\ INTEROP_TUPLE1(UnknownBaseType) diff --git a/interop/constants/typedefs.h b/interop/constants/typedefs.h index 2e36a56db..914cb27c1 100644 --- a/interop/constants/typedefs.h +++ b/interop/constants/typedefs.h @@ -22,6 +22,8 @@ namespace illumina { namespace interop { namespace constants typedef constant_type base_read_t; /** Define base type for lane metrics */ typedef constant_type base_lane_t; + /** Define base type for lane metrics */ + typedef constant_type base_run_t; }}} diff --git a/interop/io/format/metric_format.h b/interop/io/format/metric_format.h index 63a41f39b..889f9184b 100644 --- a/interop/io/format/metric_format.h +++ b/interop/io/format/metric_format.h @@ -288,7 +288,6 @@ namespace illumina { namespace interop { namespace io else { const size_t offset = metric_offset_map[metric.id()]; - INTEROP_ASSERTMSG(metric_set[offset].lane() != 0, offset); count += Layout::map_stream(in, metric_set[offset], metric_set, false); INTEROP_ASSERT(metric_set[offset].id()>0); } diff --git a/interop/io/layout/base_metric.h b/interop/io/layout/base_metric.h index ebe6c772f..807ddd50e 100644 --- a/interop/io/layout/base_metric.h +++ b/interop/io/layout/base_metric.h @@ -25,6 +25,53 @@ namespace illumina { namespace interop { namespace io { namespace layout { #pragma pack(1) + /** Empty class for InterOp records that contain run specific metrics + * + * These records contain both a lane and tile identifier. + * + * @note These classes are packed such that there is not padding. Their size reflects the accumulation of their + * member fields. + */ + struct base_run_metric + { + /** Lane integral type */ + typedef ::uint16_t lane_t; + /** Tile integral type */ + typedef ::uint32_t tile_t; + /** Define a record size type */ + typedef ::uint8_t record_size_t; + /** Define base type */ + typedef constants::base_run_t base_t; + + /** Constructor + */ + base_run_metric() : m_dummy(0) + { + } + + /** Set the lane and tile id from a base metric + * + * @param metric a base_metric from the model + */ + template + void set(const BaseMetric &/*metric*/) + { + m_dummy = 0; + } + + /** Test if the layout contains valid data + * + * @return true if data is valid + */ + bool is_valid() const + { + return true; + } + + private: + lane_t m_dummy; + }; + /** Base class for InterOp records that contain tile specific metrics * * These records contain both a lane and tile identifier. diff --git a/interop/io/metric_file_stream.h b/interop/io/metric_file_stream.h index f9bccc80c..459f5d3b0 100644 --- a/interop/io/metric_file_stream.h +++ b/interop/io/metric_file_stream.h @@ -41,14 +41,14 @@ namespace illumina { namespace interop { namespace io * @return number of bytes written */ template - size_t write_interop_to_buffer(const MetricSet& metrics, ::uint8_t* buffer, size_t buffer_size) + size_t write_interop_to_buffer(const MetricSet& metrics, ::uint8_t* buffer, const size_t buffer_size) INTEROP_THROW_SPEC((io::invalid_argument, io::bad_format_exception, io::incomplete_file_exception, io::format_exception)) { std::ostringstream fout; write_metrics(fout, metrics, metrics.version()); std::string str = fout.str(); if(buffer_size < str.length()) - INTEROP_THROW(invalid_argument, "Buffer size too small"); + INTEROP_THROW(invalid_argument, "Buffer size too small: " << buffer_size << " < " << str.length()); size_t i=0; for(;i(str[i]); @@ -63,7 +63,7 @@ namespace illumina { namespace interop { namespace io * @throw incomplete_file_exception */ template - void read_interop_from_buffer(::uint8_t* buffer, size_t buffer_size, MetricSet& metrics) INTEROP_THROW_SPEC( + void read_interop_from_buffer(::uint8_t* buffer, const size_t buffer_size, MetricSet& metrics) INTEROP_THROW_SPEC( (interop::io::file_not_found_exception, interop::io::bad_format_exception, interop::io::incomplete_file_exception, diff --git a/interop/logic/summary/summary_statistics.h b/interop/logic/summary/summary_statistics.h index 9cc2d7d20..1e7b40f12 100644 --- a/interop/logic/summary/summary_statistics.h +++ b/interop/logic/summary/summary_statistics.h @@ -148,6 +148,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary template void summarize(I beg, I end, S &stat, const bool skip_median) { + stat.clear(); if (beg == end) return; stat.mean(util::mean(beg, end)); stat.stddev(std::sqrt(util::variance_with_mean(beg, end, stat.mean()))); @@ -196,6 +197,31 @@ namespace illumina { namespace interop { namespace logic { namespace summary return size_t(std::distance(beg, end)); } + /** Calculate the sum over a collection of values, ignoring NaNs + * + * @param beg iterator to start of collection + * @param end iterator to end of collection + * @param init initial value for accumulate call + * @param op unary/binary operator for getting a value in a complex object + * @return sum of init + applying op to the range [beg, end) + */ + template + S nan_accumulate(I beg, I end, const S init, Op op) + { + S return_value = init; + if (beg == end) return init; + const float temp_value = 0; + for (; beg != end; ++beg) + { + const float current_value = op(temp_value, *beg); + if(!std::isnan(current_value)) + { + return_value += static_cast(current_value); + } + } + return return_value; + } + /** Safe divide * * @param num numerator diff --git a/interop/logic/summary/tile_summary.h b/interop/logic/summary/tile_summary.h index af3e957a1..a19350a28 100644 --- a/interop/logic/summary/tile_summary.h +++ b/interop/logic/summary/tile_summary.h @@ -89,16 +89,16 @@ namespace illumina { namespace interop { namespace logic { namespace summary util::op::const_member_function_less(&model::metrics::tile_metric::percent_pf), skip_median); stat_summary.percent_pf(stat); - stat_summary.reads(std::accumulate(tile_data.begin(), + stat_summary.reads(nan_accumulate(tile_data.begin(), tile_data.end(), - float(0), + float(0), util::op::const_member_function( &model::metrics::tile_metric::cluster_count))); - stat_summary.reads_pf(std::accumulate(tile_data.begin(), - tile_data.end(), - float(0), - util::op::const_member_function( - &model::metrics::tile_metric::cluster_count_pf))); + stat_summary.reads_pf(nan_accumulate(tile_data.begin(), + tile_data.end(), + float(0), + util::op::const_member_function( + &model::metrics::tile_metric::cluster_count_pf))); } /** Update the stat summary with cached read metrics * diff --git a/interop/model/metric_base/base_metric.h b/interop/model/metric_base/base_metric.h index 94383f48b..85ce78075 100644 --- a/interop/model/metric_base/base_metric.h +++ b/interop/model/metric_base/base_metric.h @@ -25,6 +25,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base // Forward declaration class base_metric; + class empty_metric; /** Get attributes of the metric */ template @@ -58,7 +59,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base * * @todo remove this method */ - void update_max_cycle(const base_metric &) + void update_max_cycle(const empty_metric &) { } }; @@ -68,6 +69,58 @@ namespace illumina { namespace interop { namespace model { namespace metric_base */ class empty_metric { + public: + /** id_t type */ + typedef ::uint32_t id_t; + /** Unsigned int + */ + typedef ::uint32_t uint_t; + /** Define the base type */ + typedef constants::base_run_t base_t; + + public: + /** Set the base metric identifiers + * + * @param base layout base + */ + template + void set_base(const BaseMetric &/*base*/) + { + } + /** Set id + * + * @param lane lane number + * @param tile tile number + */ + void set_base(const uint_t /*lane*/, const uint_t /*tile*/) + { + } + /** Get the metric name suffix + * + * @return empty string + */ + static const char *suffix() + { + return ""; + } + + /** Comparison operator used to sort the entries in order of their IDs + * + * @param metric2 metric to compare with the current object + * @return true if this object's ID is less than metric2's ID + */ + bool operator< (const empty_metric& /*metric2*/) const + { + return false; + } + /** Unique id created from both the lane and tile + * + * @return 1 + */ + static id_t create_id(const id_t, const id_t, const id_t= 0)// TODO: remove hack (const id_t=0) + { + return 1;// Cannot be zero + } }; /** Base class for InterOp classes that contain tile specific metrics diff --git a/interop/model/metric_base/metric_set.h b/interop/model/metric_base/metric_set.h index a4dcfe946..8edea37c2 100644 --- a/interop/model/metric_base/metric_set.h +++ b/interop/model/metric_base/metric_set.h @@ -30,6 +30,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { + /** Compare metrics across type */ + template struct metric_comparison; /** Metric set holds a collection metrics * * This class holds a map that maps a unique id to the metric. @@ -69,6 +71,8 @@ namespace illumina { namespace interop { namespace model { namespace metric_base #endif public: + /** Define a safe comparison for ids */ + typedef metric_comparison metric_comparison_t; /** Const metric iterator */ typedef typename metric_array_t::const_iterator const_iterator; /** Metric iterator */ @@ -134,9 +138,9 @@ namespace illumina { namespace interop { namespace model { namespace metric_base reserve(origin.size()+size()); for(const_iterator it = origin.begin();it != origin.end();++it) { - if(it->lane() != tile_id.lane() || it->tile() != tile_id.tile()) - continue; - insert(*it); + if(metric_comparison_t::to_lane(tile_id) == to_lane(*it) && + metric_comparison_t::to_tile(tile_id) == to_tile(*it)) + insert(*it); } } /** Flag that indicates whether the data source exists @@ -390,7 +394,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { size_t lane_max = 0; for (const_iterator b = begin(); b != end(); ++b) - lane_max = std::max(lane_max, static_cast(b->lane())); + lane_max = std::max(lane_max, static_cast(to_lane(*b))); return lane_max; } @@ -755,17 +759,17 @@ namespace illumina { namespace interop { namespace model { namespace metric_base private: static id_t to_id(const metric_type &metric) { - return metric.id(); + return metric_comparison_t::to_id(metric); } static uint_t to_lane(const metric_type &metric) { - return metric.lane(); + return metric_comparison_t::to_lane(metric); } static uint_t to_tile(const metric_type &metric) { - return metric.tile(); + return metric_comparison_t::to_tile(metric); } void cycles(id_set_t& cycles_set) const @@ -829,7 +833,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base { } bool operator()(const metric_type &metric) const - { return metric.lane() == m_lane; } + { return metric_comparison_t::to_lane(metric) == m_lane; } const uint_t m_lane; }; @@ -878,6 +882,57 @@ namespace illumina { namespace interop { namespace model { namespace metric_base offset_map_t m_id_map; }; + template + struct metric_comparison + { + /** Define a ID type */ + typedef typename Metric::id_t id_t; + /** Define a lane/tile/cycle id type */ + typedef typename Metric::uint_t uint_t; + template + static id_t to_id(const MetricT &metric) + { + return metric.id(); + } + + template + static uint_t to_lane(const MetricT &metric) + { + return metric.lane(); + } + + template + static uint_t to_tile(const MetricT &metric) + { + return metric.tile(); + } + }; + template + struct metric_comparison + { + /** Define a ID type */ + typedef typename Metric::id_t id_t; + /** Define a lane/tile/cycle id type */ + typedef typename Metric::uint_t uint_t; + template + static id_t to_id(const MetricT &) + { + return 1;// Cannot be zero + } + + template + static uint_t to_lane(const MetricT &) + { + return 1; + } + + template + static uint_t to_tile(const MetricT &) + { + return 1; + } + }; + /** Get metric set for a given metric set */ template struct metric_set_helper diff --git a/interop/model/metrics/summary_run_metric.h b/interop/model/metrics/summary_run_metric.h new file mode 100644 index 000000000..dd244f778 --- /dev/null +++ b/interop/model/metrics/summary_run_metric.h @@ -0,0 +1,212 @@ +/** Summary run metric + * + * The summary run metric contains run-level summary information + * + * @file + * @date 07/06/2020 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once + +#include "interop/io/format/generic_layout.h" +#include "interop/model/metric_base/base_metric.h" +#include "interop/model/metric_base/metric_set.h" + +namespace illumina { namespace interop { namespace model { namespace metrics +{ + + /** Summary run metric + * + * The summary run metric contains run-level summary information + * + * @note Supported versions: 1 + */ + class summary_run_metric : public metric_base::empty_metric + { + public: + enum + { + /** Unique type code for metric */ + TYPE = constants::SummaryRun, + /** Latest version of the InterOp format */ + LATEST_VERSION = 1 + }; + /** Static run metric header */ + typedef metric_base::base_metric_header header_type; + /** Vector of floats */ + typedef std::vector float_vector; + /** ubyte_t type */ + typedef uint8_t ubyte_t; + /** ushort_t type */ + typedef uint16_t ushort_t; + /** uint_t type */ + typedef uint32_t uint_t; + /** Count type */ + typedef double count_t; + + public: + /** Constructor */ + summary_run_metric() : + m_occupancy_proxy_cluster_count(missing()) + , m_raw_cluster_count(missing()) + , m_occupied_cluster_count(missing()) + , m_pf_cluster_count(missing()) + { } + /** Constructor */ + summary_run_metric(const header_type&) : + m_occupancy_proxy_cluster_count(missing()) + , m_raw_cluster_count(missing()) + , m_occupied_cluster_count(missing()) + , m_pf_cluster_count(missing()) + { } + + /** Constructor + * + * @note Version 1 + * @param occupancy_proxy_cluster_count proxy for occupancy + * @param raw_cluster_count raw cluster count + * @param occupied_cluster_count occupied cluster count + * @param pf_cluster_count pf cluster count + */ + summary_run_metric(const count_t occupancy_proxy_cluster_count + , const count_t raw_cluster_count + , const count_t occupied_cluster_count + , const count_t pf_cluster_count + ) : + m_occupancy_proxy_cluster_count(occupancy_proxy_cluster_count) + , m_raw_cluster_count(raw_cluster_count) + , m_occupied_cluster_count(occupied_cluster_count) + , m_pf_cluster_count(pf_cluster_count) + {} + + public: + /** Setter + * + * @note Version 1 + * @param occupancy_proxy_cluster_count proxy for occupancy + * @param raw_cluster_count raw cluster count + * @param occupied_cluster_count occupied cluster count + * @param pf_cluster_count pf cluster count + */ + void set(const count_t occupancy_proxy_cluster_count + , const count_t raw_cluster_count + , const count_t occupied_cluster_count + , const count_t pf_cluster_count) + { + m_occupancy_proxy_cluster_count = occupancy_proxy_cluster_count; + m_raw_cluster_count = raw_cluster_count; + m_occupied_cluster_count = occupied_cluster_count; + m_pf_cluster_count = pf_cluster_count; + } + + /** @defgroup summary_run_metric Run distortion metric + * + * Run summary metrics + * + * @ref illumina::interop::model::metrics::summary_run_metric "See full class description" + * @ingroup run_metrics + * @{ + */ + /** Return raw cluster count + * + * @return raw cluster count + */ + count_t raw_cluster_count() const + { + return m_raw_cluster_count; + } + /** Return occupancy cluster count + * + * @return occupancy cluster count + */ + count_t occupied_cluster_count() const + { + return m_occupied_cluster_count; + } + /** Return PF cluster count + * + * @return PF cluster count + */ + count_t pf_cluster_count() const + { + return m_pf_cluster_count; + } + /** Return occupancy proxy cluster count + * + * @return occupancy proxy cluster count + */ + count_t occupancy_proxy_cluster_count() const + { + return m_occupancy_proxy_cluster_count; + } + /** Return percent occupancy proxy + * + * @return percent occupancy proxy + */ + count_t percent_occupancy_proxy() const + { + if(std::isnan(m_pf_cluster_count)) return missing(); + if(std::isnan(m_occupancy_proxy_cluster_count)) return missing(); + return m_occupancy_proxy_cluster_count / m_pf_cluster_count * 100.0; + } + /** Return percent PF + * + * @return percent PF + */ + count_t percent_pf() const + { + if(std::isnan(m_raw_cluster_count)) return missing(); + if(std::isnan(m_pf_cluster_count)) return missing(); + return m_pf_cluster_count / m_raw_cluster_count * 100.0; + } + /** Return percent occupancy + * + * @return percent occupancy + */ + count_t percent_occupied() const + { + if(std::isnan(m_raw_cluster_count)) return missing(); + if(std::isnan(m_occupied_cluster_count)) return missing(); + return m_occupied_cluster_count / m_raw_cluster_count * 100.0; + } + /** @} */ + /** Unique id created from both the lane and tile + * + * @return unique identifier + */ + id_t id() const + { + return 1;// There is only ever 1 per run, cannot be zero + } + + public: + /** Get the prefix of the InterOp filename + * + * @return prefix + */ + static const char *prefix() + { return "SummaryRun"; } + + public: + /** Flag if value is missing + * + * @return missing sentinel + */ + static count_t missing() + { + return std::numeric_limits::quiet_NaN(); + } + + private: + count_t m_occupancy_proxy_cluster_count; + count_t m_raw_cluster_count; + count_t m_occupied_cluster_count; + count_t m_pf_cluster_count; + + template + friend + struct io::generic_layout; + }; +}}}} + diff --git a/interop/model/run_metrics.h b/interop/model/run_metrics.h index c38655f36..b5d64eaee 100644 --- a/interop/model/run_metrics.h +++ b/interop/model/run_metrics.h @@ -29,6 +29,7 @@ #include "interop/model/metrics/q_by_lane_metric.h" #include "interop/model/metrics/q_collapsed_metric.h" #include "interop/model/metrics/tile_metric.h" +#include "interop/model/metrics/summary_run_metric.h" namespace illumina { namespace interop { namespace model { namespace metrics { @@ -48,6 +49,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @see tile_metrics * @see q_by_lane_metric * @see q_collapsed_metric + * @see summary_run_metric */ class run_metrics { @@ -65,7 +67,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics q_metric, q_by_lane_metric, q_collapsed_metric, - tile_metric + tile_metric, + summary_run_metric >::result_t metric_type_list_t; private: @@ -427,7 +430,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param buffer_size size of binary buffer */ void read_metrics_from_buffer(const constants::metric_group group, - uint8_t* buffer, + ::uint8_t* buffer, const size_t buffer_size) INTEROP_THROW_SPEC(( io::file_not_found_exception, io::bad_format_exception, @@ -440,7 +443,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param buffer_size size of binary buffer */ void write_metrics_to_buffer(const constants::metric_group group, - uint8_t* buffer, + ::uint8_t* buffer, const size_t buffer_size)const INTEROP_THROW_SPEC(( io::invalid_argument, io::bad_format_exception, diff --git a/interop/model/summary/metric_summary.h b/interop/model/summary/metric_summary.h index b3843e410..7ebc42202 100644 --- a/interop/model/summary/metric_summary.h +++ b/interop/model/summary/metric_summary.h @@ -26,7 +26,8 @@ namespace illumina { namespace interop { namespace model { namespace summary { m_percent_gt_q30(std::numeric_limits::quiet_NaN()), m_yield_g(std::numeric_limits::quiet_NaN()), m_projected_yield_g(0), - m_percent_occupied(std::numeric_limits::quiet_NaN()) + m_percent_occupied(std::numeric_limits::quiet_NaN()), + m_percent_occupancy_proxy(std::numeric_limits::quiet_NaN()) {} public: @@ -103,6 +104,16 @@ namespace illumina { namespace interop { namespace model { namespace summary { { return m_percent_occupied; } + /** Get the percent occupancy proxy + * + * Also known as % loading concentration + * + * @return percent occupancy proxy + */ + float percent_occupancy_proxy()const + { + return m_percent_occupancy_proxy; + } /** @} */ /** Set the first cycle intensity * @@ -161,6 +172,16 @@ namespace illumina { namespace interop { namespace model { namespace summary { { m_percent_occupied = val; } + /** Set the percent occupancy proxy + * + * Also known as % loading concentration + * + * @param val percent occupancy proxy + */ + void percent_occupancy_proxy(const float val) + { + m_percent_occupancy_proxy = val; + } /** Resize the underlying data */ @@ -176,6 +197,7 @@ namespace illumina { namespace interop { namespace model { namespace summary { float m_yield_g; float m_projected_yield_g; float m_percent_occupied; + float m_percent_occupancy_proxy; template friend struct io::generic_layout; }; diff --git a/interop/model/summary/stat_summary.h b/interop/model/summary/stat_summary.h index 038904e0a..55995703f 100644 --- a/interop/model/summary/stat_summary.h +++ b/interop/model/summary/stat_summary.h @@ -273,7 +273,6 @@ namespace illumina { namespace interop { namespace model { namespace summary } /** Get mean summarizing the percent occupied * - * @note IUO * @return statistics summarizing the percent occupied */ const metric_stat_t &percent_occupied() const @@ -470,7 +469,6 @@ namespace illumina { namespace interop { namespace model { namespace summary } /** Set mean summarizing the percent occupied * - * @note IUO * @param val statistics summarizing the percent occupied */ void percent_occupied(const metric_stat_t& val) diff --git a/interop/util/exception_specification.h b/interop/util/exception_specification.h index b9a002a7f..a1f9157a5 100644 --- a/interop/util/exception_specification.h +++ b/interop/util/exception_specification.h @@ -5,7 +5,7 @@ * @file * @date 6/25/18 * @version 1.0 - * @copyright Illumina Use Only + * @copyright GNU Public License. */ #pragma once diff --git a/interop/util/type_traits.h b/interop/util/type_traits.h index 9cd08c64c..6b9302e8e 100644 --- a/interop/util/type_traits.h +++ b/interop/util/type_traits.h @@ -31,12 +31,13 @@ namespace illumina { namespace interop typename T9=null_type, typename T10=null_type, typename T11=null_type, typename T12=null_type, typename T13=null_type, typename T14=null_type, typename T15=null_type, typename T16=null_type, typename T17=null_type, typename T18=null_type, typename T19=null_type, typename T20=null_type, - typename T21=null_type, typename T22=null_type, typename T23=null_type, typename T24=null_type> + typename T21=null_type, typename T22=null_type, typename T23=null_type, typename T24=null_type, + typename T25=null_type, typename T26=null_type> struct make_type_list { private: typedef typename make_type_list< - T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24 + T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24, T25, T26 >::result_t tail_result_t; public: /** List of types */ diff --git a/src/apps/aggregate.cpp b/src/apps/aggregate.cpp index be09bd0fa..72b15a002 100644 --- a/src/apps/aggregate.cpp +++ b/src/apps/aggregate.cpp @@ -63,6 +63,8 @@ struct subset_copier m_run.get().insert(metrics[i]); } } + template + void copy(const MetricSet&, const constants::base_run_t *)const{} private: run_metrics& m_run; diff --git a/src/ext/swig/arrays/arrays_numpy_impl.i b/src/ext/swig/arrays/arrays_numpy_impl.i index 894917795..ef96487f6 100644 --- a/src/ext/swig/arrays/arrays_numpy_impl.i +++ b/src/ext/swig/arrays/arrays_numpy_impl.i @@ -7,8 +7,8 @@ import_array(); %} -%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(::uint8_t* buffer, size_t buffer_size)} -%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(unsigned char* buffer, size_t buffer_size)} +%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(::uint8_t* buffer, const size_t buffer_size)} +%apply (unsigned char* INPLACE_ARRAY1, int DIM1) {(unsigned char* buffer, const size_t buffer_size)} %apply (float* INPLACE_ARRAY1, int DIM1) {(float* buffer, size_t buffer_size)} %apply (unsigned int* INPLACE_ARRAY1, int DIM1) {(::uint32_t* id_buffer, size_t id_buffer_size)} %apply (float* INPLACE_ARRAY1, int DIM1) {(float* data_beg, const size_t n)} diff --git a/src/ext/swig/metrics.i b/src/ext/swig/metrics.i index ef1f11f41..723f969de 100644 --- a/src/ext/swig/metrics.i +++ b/src/ext/swig/metrics.i @@ -140,6 +140,7 @@ METRICS_EXCEPTION_WRAPPER(WRAP_EXCEPTION) WRAPPER(index_metric) WRAPPER(q_collapsed_metric) WRAPPER(q_by_lane_metric) + WRAPPER(summary_run_metric) %enddef diff --git a/src/interop/CMakeLists.txt b/src/interop/CMakeLists.txt index 94f6ce4b5..3822369b7 100644 --- a/src/interop/CMakeLists.txt +++ b/src/interop/CMakeLists.txt @@ -12,6 +12,7 @@ set(SRCS model/metrics/q_metric.cpp model/metrics/index_metric.cpp model/metrics/q_collapsed_metric.cpp + model/metrics/summary_run_metric.cpp model/summary/run_summary.cpp logic/plot/plot_by_cycle.cpp logic/plot/plot_by_lane.cpp @@ -35,7 +36,8 @@ set(SRCS logic/plot/plot_metric_list.cpp logic/metric/index_metric.cpp model/metrics/extended_tile_metric.cpp - logic/metric/extended_tile_metric.cpp) + logic/metric/extended_tile_metric.cpp + ) set(HEADERS ../../interop/io/paths.h @@ -59,6 +61,7 @@ set(HEADERS ../../interop/model/run/read_info.h ../../interop/model/metrics/tile_metric.h ../../interop/model/metrics/corrected_intensity_metric.h + ../../interop/model/metrics/summary_run_metric.h ../../interop/io/layout/base_metric.h ../../interop/model/metric_base/metric_set.h ../../interop/model/metric_base/base_metric.h diff --git a/src/interop/logic/summary/run_summary.cpp b/src/interop/logic/summary/run_summary.cpp index 0b204c9d3..146c01458 100644 --- a/src/interop/logic/summary/run_summary.cpp +++ b/src/interop/logic/summary/run_summary.cpp @@ -14,6 +14,7 @@ #include "interop/logic/metric/q_metric.h" #include "interop/logic/summary/phasing_summary.h" #include "interop/logic/metric/dynamic_phasing_metric.h" +#include "interop/model/metrics/summary_run_metric.h" namespace illumina { namespace interop { namespace logic { namespace summary @@ -99,6 +100,13 @@ namespace illumina { namespace interop { namespace logic { namespace summary } } + void set_run_summary_metric(const model::metrics::summary_run_metric &summary_run, model::summary::metric_summary& summary) + { + summary.percent_occupied(static_cast(summary_run.percent_occupied())); + summary.percent_occupancy_proxy(static_cast(summary_run.percent_occupancy_proxy())); + } + + /** Summarize a collection run metrics * * TODO speed up calculation by adding no_median flag @@ -198,6 +206,16 @@ namespace illumina { namespace interop { namespace logic { namespace summary naming_method, skip_median); + if(!metrics.get().empty()) + { + const summary_run_metric &summary_run = metrics.get().at(0); + set_run_summary_metric(summary_run, summary.nonindex_summary()); + set_run_summary_metric(summary_run, summary.total_summary()); + for (size_t read = 0; read < summary.size(); ++read) + { + set_run_summary_metric(summary_run, summary[read].summary()); + } + } if(trim) { // Remove the empty lane summary entries diff --git a/src/interop/model/metrics/summary_run_metric.cpp b/src/interop/model/metrics/summary_run_metric.cpp new file mode 100644 index 000000000..6577bfbab --- /dev/null +++ b/src/interop/model/metrics/summary_run_metric.cpp @@ -0,0 +1,192 @@ +/** Register format layouts for summary run metrics + * + * Each version of the summary run metrics file has a layout defined below. + * + * @file + * @date 7/06/2020 + * @version 1.0 + * @copyright GNU Public License. + */ + +#include +#include +#include "interop/model/metrics/summary_run_metric.h" +#include "interop/io/format/metric_format_factory.h" +#include "interop/io/format/text_format_factory.h" +#include "interop/io/format/text_format.h" +#include "interop/io/format/default_layout.h" +#include "interop/io/format/metric_format.h" + +using namespace illumina::interop::model::metrics; + +namespace illumina { namespace interop { namespace io +{ +#pragma pack(1) + + /** Summary Run Metric Record Layout Version 1 + * + * This class provides an interface to reading the summary run metric file: + * - InterOp/SummaryRun.bin + * - InterOp/SummaryRunOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: summary_run_metric + * 2. Version: 1 + */ + template<> + struct generic_layout : public default_layout<1> + { + /** @page summary_run_v1 SummaryRun Version 1 + * + * This class provides an interface to reading the SummaryRun metric file: + * - InterOp/SummaryRun.bin + * - InterOp/SummaryRunOut.bin + * + * The file format for SummaryRun metrics is as follows: + * + * @b Header + * + * illumina::interop::io::metric_format_stream (Class that parses this information) + * + * byte 0: version number (1) + * + * @b n-Records + * + * illumina::interop::io::generic_layout (Class that parses this information) + * 2 byte: dummy (int16) + * 4 byte: occupancy proxy cluster count (float) + * 4 byte: raw cluster count (float) + * 4 byte: occupancy cluster count (float) + * 4 byte: PF cluster count (float) + */ + + /** Metric ID type */ + typedef layout::base_run_metric metric_id_t; + + /** Define a record size type */ + typedef ::uint32_t record_size_t; + + /**Count type */ + typedef summary_run_metric::count_t count_t; + + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @param header metric header layout + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream &stream, Metric &metric, Header &/*header*/, const bool) + { + std::streamsize count = 0; + count += stream_map< count_t >(stream, metric.m_occupancy_proxy_cluster_count); + count += stream_map< count_t >(stream, metric.m_raw_cluster_count); + count += stream_map< count_t >(stream, metric.m_occupied_cluster_count); + count += stream_map< count_t >(stream, metric.m_pf_cluster_count); + return count; + } + + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const summary_run_metric::header_type &/*header*/) + { + const size_t metric_id_byte_count = sizeof(metric_id_t); + return static_cast( + metric_id_byte_count + + sizeof(count_t) + + sizeof(count_t) + + sizeof(count_t) + + sizeof(count_t) + ); + } + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const summary_run_metric::header_type&) + { + return static_cast(sizeof(record_size_t) + sizeof(version_t)); + } + }; + +#pragma pack() // DO NOT MOVE + /** Static Run Metric CSV text format + * + * This class provide an interface for writing the summary run metrics to a CSV file: + * + * - SummaryRun.csv + */ + template<> + struct text_layout< summary_run_metric, 1 > + { + /** Define a header type */ + typedef summary_run_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param header metric set header + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type& /*header*/, + const std::vector&, + const char sep, + const char eol) + { + const char* column_headers[] = + { + "Raw Cluster Count", "Occupied Cluster Count", "PF Cluster Count", "Occupancy Proxy Cluster Count" + }; + std::vector headers; + headers.reserve(util::length_of(column_headers)); + for(size_t i=0;i + void determine(const MetricSet &, const constants::base_run_t *) + { + } private: constants::tile_naming_method m_naming_method; @@ -229,7 +233,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics { public: read_metric_set_from_binary_buffer(const constants::metric_group group, - uint8_t* buffer, + ::uint8_t* buffer, const size_t buffer_size) : m_group(group), m_buffer(buffer), @@ -747,7 +751,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param buffer_size size of binary buffer */ void run_metrics::read_metrics_from_buffer(const constants::metric_group group, - uint8_t* buffer, + ::uint8_t* buffer, const size_t buffer_size) INTEROP_THROW_SPEC(( io::file_not_found_exception, io::bad_format_exception, @@ -763,7 +767,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param buffer_size size of binary buffer */ void run_metrics::write_metrics_to_buffer(const constants::metric_group group, - uint8_t* buffer, + ::uint8_t* buffer, const size_t buffer_size)const INTEROP_THROW_SPEC(( io::invalid_argument, io::bad_format_exception, diff --git a/src/interop/model/run_metrics_helper.cpp b/src/interop/model/run_metrics_helper.cpp index 6ab94861e..4a14300eb 100644 --- a/src/interop/model/run_metrics_helper.cpp +++ b/src/interop/model/run_metrics_helper.cpp @@ -31,6 +31,8 @@ namespace illumina { namespace interop { namespace model { namespace metrics } } + template + void populate_id(const MetricSet &, const constants::base_run_t *) const {} template void populate_id(const MetricSet &, const constants::base_lane_t *) const {} diff --git a/src/tests/interop/CMakeLists.txt b/src/tests/interop/CMakeLists.txt index aba151add..1027fce06 100644 --- a/src/tests/interop/CMakeLists.txt +++ b/src/tests/interop/CMakeLists.txt @@ -20,6 +20,7 @@ set(SRCS metrics/tile_metrics_test.cpp metrics/q_by_lane_metric_test.cpp metrics/q_collapsed_metrics_test.cpp + metrics/summary_run_metrics_test.cpp metrics/base_metric_tests.cpp metrics/run_metric_test.cpp metrics/metric_streams_test.cpp @@ -56,6 +57,7 @@ set(HEADERS metrics/inc/tile_metrics_test.h metrics/inc/q_collapsed_metrics_test.h metrics/inc/extended_tile_metrics_test.h + metrics/inc/summary_run_metrics_test.h inc/failure_listener.h inc/persistent_parameter_generator.h inc/generic_fixture.h diff --git a/src/tests/interop/metrics/extended_tile_metrics_test.cpp b/src/tests/interop/metrics/extended_tile_metrics_test.cpp index 1b1eba069..c814cad57 100644 --- a/src/tests/interop/metrics/extended_tile_metrics_test.cpp +++ b/src/tests/interop/metrics/extended_tile_metrics_test.cpp @@ -25,12 +25,10 @@ typedef metric_set< extended_tile_metric > extended_tile_metric_set; struct extended_tile_metrics_tests : public generic_test_fixture< extended_tile_metric_set > {}; extended_tile_metrics_tests::generator_type extended_tile_unit_test_generators[] = { - // IUO wrap(new hardcoded_metric_generator< extended_tile_metric_v1 >), wrap(new write_read_metric_generator< extended_tile_metric_v1 >), wrap(new by_cycle_metric_generator< extended_tile_metric_v1 >), wrap(new clear_metric_generator< extended_tile_metric_v1 >), - // End IUO wrap(new hardcoded_metric_generator< extended_tile_metric_v2 >), wrap(new by_cycle_metric_generator< extended_tile_metric_v2 >), wrap(new write_read_metric_generator< extended_tile_metric_v2 >), diff --git a/src/tests/interop/metrics/inc/metric_format_fixtures.h b/src/tests/interop/metrics/inc/metric_format_fixtures.h index 7a938e911..1fb6c54dd 100644 --- a/src/tests/interop/metrics/inc/metric_format_fixtures.h +++ b/src/tests/interop/metrics/inc/metric_format_fixtures.h @@ -19,6 +19,7 @@ #include "src/tests/interop/metrics/inc/q_metrics_test.h" #include "src/tests/interop/metrics/inc/tile_metrics_test.h" #include "src/tests/interop/metrics/inc/extended_tile_metrics_test.h" +#include "src/tests/interop/metrics/inc/summary_run_metrics_test.h" namespace illumina{ namespace interop { namespace unittest { @@ -57,7 +58,8 @@ namespace illumina{ namespace interop { namespace unittest q_metric_v6_unbinned, q_metric_v7, tile_metric_v2, - tile_metric_v3 + tile_metric_v3, + summary_run_v1 > PublicFormats; diff --git a/src/tests/interop/metrics/inc/metric_generator.h b/src/tests/interop/metrics/inc/metric_generator.h index 4c62f92b4..c41664b18 100644 --- a/src/tests/interop/metrics/inc/metric_generator.h +++ b/src/tests/interop/metrics/inc/metric_generator.h @@ -162,6 +162,7 @@ namespace illumina{ namespace interop { namespace unittest { typedef typename metric_set_t::metric_type metric_t; typedef typename metric_t::uint_t uint_t; + typedef typename metric_set_t::metric_comparison_t metric_comparison_t; std::string expected_data; { @@ -181,10 +182,10 @@ namespace illumina{ namespace interop { namespace unittest metric_t tmp = expected[0]; const size_t n = expected.size(); for(size_t i=0;i +#include "metric_test.h" +#include "interop/model/metrics/summary_run_metric.h" +#include "interop/model/summary/run_summary.h" + +namespace illumina{ namespace interop { namespace unittest +{ + + /** This test writes three records of an InterOp files, then reads them back in and compares + * each value to ensure they did not change. + * + * @see model::metrics::summary_run_metric + * @note Version 1 + */ + struct summary_run_v1 : metric_test + { + /** Create the expected metric set + * + * @param metrics destination metric set + */ + static void create_expected(metric_set_t &metrics, const model::run::info& =model::run::info()) + { + metrics = metric_set_t(header_t(), VERSION); + metrics.insert(metric_t(1.0f, 2.0f, 3.0f, 4.0f));// Only supports single metric! + } + /** Get the expected binary data + * + * @param buffer binary data string + */ + template + static void create_binary_data(Collection &buffer) + { + const int tmp[] = + { + 1 + ,34,0,0,0,0,0,0,0,0,0,0,0,-16,63,0,0,0,0,0,0 + ,0,64,0,0,0,0,0,0,8,64,0,0,0,0,0,0,16,64 + }; + buffer.assign(tmp, tmp+util::length_of(tmp)); + } + }; + +}}} + diff --git a/src/tests/interop/metrics/metric_streams_test.cpp b/src/tests/interop/metrics/metric_streams_test.cpp index 35452a6d7..23a69bb44 100644 --- a/src/tests/interop/metrics/metric_streams_test.cpp +++ b/src/tests/interop/metrics/metric_streams_test.cpp @@ -15,6 +15,7 @@ #include "interop/io/metric_stream.h" #include "interop/io/metric_file_stream.h" #include "src/tests/interop/metrics/inc/metric_format_fixtures.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop; using namespace illumina::interop::unittest; diff --git a/src/tests/interop/metrics/run_metric_test.cpp b/src/tests/interop/metrics/run_metric_test.cpp index c9a4b29a1..f4bcff463 100644 --- a/src/tests/interop/metrics/run_metric_test.cpp +++ b/src/tests/interop/metrics/run_metric_test.cpp @@ -99,6 +99,9 @@ TEST(run_metric_test, summary_subset_of_imaging) ASSERT_EQ(load_imaging.size(), load_summary.size()); for(size_t i=0;i(i) == static_cast(model::metrics::summary_run_metric::TYPE)) + continue; EXPECT_TRUE(load_summary[i] == 0 || load_summary[i] == load_imaging[i]) << constants::to_string(static_cast(i)); } @@ -107,15 +110,18 @@ TEST(run_metric_test, summary_subset_of_imaging) TYPED_TEST_P(run_metric_test, append_tiles) { typedef typename TestFixture::metric_set_t metric_set_t; + typedef typename metric_set_t::base_t base_t; + typedef typename metric_set_t::metric_comparison_t metric_comparison_t; + + if(base_t::value() == constants::BaseRunType) return; metric_set_t& metric_set = TestFixture::expected. template get(); const model::metric_base::base_metric tile_id; - //tile_id. - //= metric_set[0]; size_t count_expected = 0; for(size_t i=0;i +#include +#include "interop/model/run_metrics.h" +#include "src/tests/interop/metrics/inc/summary_run_metrics_test.h" +#include "src/tests/interop/inc/generic_fixture.h" +#include "src/tests/interop/inc/proxy_parameter_generator.h" +#include "src/tests/interop/metrics/inc/metric_generator.h" +using namespace illumina::interop::model::metrics; +using namespace illumina::interop::model::metric_base; +using namespace illumina::interop::io; +using namespace illumina::interop::unittest; +using namespace illumina::interop; + + +typedef metric_set< summary_run_metric > summary_run_metric_set; +/** Setup for tests that compare two metric sets */ +struct summary_run_metrics_tests : public generic_test_fixture< summary_run_metric_set > {}; + +/** This does not change per cycle */ +by_cycle_metric_generator< summary_run_v1 > summary_run_dummy_v1; + + +summary_run_metrics_tests::generator_type summary_run_unit_test_generators[] = { + wrap(new hardcoded_metric_generator< summary_run_v1 >), + wrap(new write_read_metric_generator< summary_run_v1 >), + wrap(new clear_metric_generator< summary_run_v1 >) +}; + +// Setup unit tests for summary_run_metrics_tests +INSTANTIATE_TEST_CASE_P(summary_run_metric_unit_test, + summary_run_metrics_tests, + ::testing::ValuesIn(summary_run_unit_test_generators)); + +/** + * @class illumina::interop::model::metrics::summary_run_metric + * @test Confirm version 1 of the metric can be written to and read from a stream + * @test Confirm version 1 of the metric matches known binary file + */ +TEST_P(summary_run_metrics_tests, test_read_write) +{ + typedef summary_run_metric_set::const_iterator const_iterator; + if(skip_test) return; + ASSERT_TRUE(fixture_test_result); + EXPECT_EQ(actual.version(), expected.version()); + EXPECT_EQ(actual.size(), expected.size()); + const float eps = 1e-7f; + for (const_iterator it_expected = expected.begin(), it_actual = actual.begin(); + it_expected != expected.end() && it_actual != actual.end(); + it_expected++, it_actual++) + { + EXPECT_NEAR(it_expected->raw_cluster_count(), it_actual->raw_cluster_count(), eps); + EXPECT_NEAR(it_expected->occupied_cluster_count(), it_actual->occupied_cluster_count(), eps); + EXPECT_NEAR(it_expected->pf_cluster_count(), it_actual->pf_cluster_count(), eps); + EXPECT_NEAR(it_expected->occupancy_proxy_cluster_count(), it_actual->occupancy_proxy_cluster_count(), eps); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Setup regression test +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +regression_test_metric_generator summary_run_regression_gen; +INSTANTIATE_TEST_CASE_P(summary_run_metric_regression_test, + summary_run_metrics_tests, + ProxyValuesIn(summary_run_regression_gen, regression_test_data::instance().files())); + diff --git a/src/tests/python/CoreTests.py b/src/tests/python/CoreTests.py index 2997d761a..ff0a49b49 100644 --- a/src/tests/python/CoreTests.py +++ b/src/tests/python/CoreTests.py @@ -196,7 +196,27 @@ def test_invalid_argument(self): py_interop_comm.write_interop_to_buffer(run.extraction_metric_set(), buf) self.fail("invalid_argument should have been thrown") except py_interop_comm.invalid_argument as ex: - self.assertEqual(str(ex).split('\n')[0], "Buffer size too small") + self.assertEqual(str(ex).split('\n')[0], "Buffer size too small: 3 < 116") + + def test_invalid_argument_run_metrics_read(self): + """ + Test that exceptions can be caught and they have the expected message + """ + + tmp = numpy.asarray( + [2,38 + ,7,0,90,4,1,0,-12,-56,15,64,-98,35,12,64,0,0,0,0,0,0,0,0,46,1,17,1,0,0,0,0,96,-41,-104,36,122,-86,-46,-120 + ,7,0,-66,4,1,0,96,-43,14,64,-63,49,13,64,0,0,0,0,0,0,0,0,56,1,17,1,0,0,0,0,112,125,77,38,122,-86,-46,-120 + ,7,0,66,8,1,0,74,-68,6,64,-118,-7,8,64,0,0,0,0,0,0,0,0,93,1,46,1,0,0,0,0,-47,-104,2,40,122,-86,-46,-120], + dtype=numpy.uint8) + run = py_interop_run_metrics.run_metrics() + run.read_metrics_from_buffer(py_interop_run.Extraction, tmp) + try: + buf = numpy.zeros(3, dtype=numpy.uint8) + py_interop_comm.write_interop_to_buffer(run.extraction_metric_set(), buf) + self.fail("invalid_argument should have been thrown") + except py_interop_comm.invalid_argument as ex: + self.assertEqual(str(ex).split('\n')[0], "Buffer size too small: 3 < 116") def test_invalid_filter_option(self): """ @@ -236,7 +256,7 @@ def test_invalid_parameter(self): run_metrics.read_metrics("", 3, valid_to_load, 1) self.fail("invalid_parameter should have been thrown") except py_interop_run_metrics.invalid_parameter as ex: - self.assertEqual(str(ex).split('\n')[0], "Boolean array valid_to_load does not match expected number of metrics: 2 != 12") + self.assertEqual(str(ex).split('\n')[0], "Boolean array valid_to_load does not match expected number of metrics: 2 != 13") def test_invalid_metric_type(self): """ @@ -402,6 +422,14 @@ def test_to_string_metric_group(self): self.assertEqual(py_interop_run.to_string_metric_group(py_interop_run.Error), "Error") + def test_summary_run_metric_set_method(self): + """ + Test if the enum parsing is properly wrapped + """ + + run = py_interop_run_metrics.run_metrics() + summary_run_metric_set = run.summary_run_metric_set() + if __name__ == '__main__': unittest.main()