Issue-?: Support % loading concentration (#236)

Bump tests

Bump tests

Fix another warning

Fix warnings

Fix Python unit test

Fix more

Fix IUO issues

docs/src/binary_formats.md

@@ -44,6 +44,7 @@ The documentation for the model notes when an attribute is only populated by a s
  - @subpage q_collapsed_v4 "Collapsed Q-Metrics Version 4"
  - @subpage q_collapsed_v5 "Collapsed Q-Metrics Version 5"
  - @subpage q_collapsed_v6 "Collapsed Q-Metrics Version 6"
+ - @subpage summary_run_v1 "Summary Run Version 1"
 
  The following are binary formats used only for testing purposes and are not officially supported:
 

docs/src/changes.md

@@ -1,5 +1,13 @@
 # Changes                                               {#changes}
-
+
+## v1.1.15
+
+Date       | Description
+---------- | -----------
+2020-08-26 | Issue-229: Support % loading concentration
+2020-08-26 | Issue-229: Fix Python binding for read_metrics_from_buffer
+
+
 ## v1.1.14
 
 Date       | Description

docs/src/index.md

@@ -58,6 +58,7 @@ List of InterOp Metric Files
 | [QMetricsByLaneOut.bin]          | Per tile per cycle Q-score histogram per lane                                     |
 | [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle                                                |
 | [ExtendedTileMetricsOut.bin]     | Per tile occupancy metrics                                                        |
+| [SummaryRunMetricsOut.bin]       | Per run summary metrics                                                           |
 
 [CorrectedIntMetricsOut.bin]: @ref corrected_intensity "CorrectedIntMetricsOut.bin"
 [ErrorMetricsOut.bin]: @ref error_metric "ErrorMetricsOut.bin"
@@ -70,6 +71,7 @@ List of InterOp Metric Files
 [QMetricsByLaneOut.bin]: @ref q_metric_by_lane "QMetricsByLaneOut.bin"
 [EmpiricalPhasingMetricsOut.bin]: @ref phasing_metric "EmpiricalPhasingMetricOut.bin"
 [ExtendedTileMetricsOut.bin]: @ref extended_tile_metric "ExtendedTileMetricsOut.bin"
+[SummaryRunMetricsOut.bin]: @ref summary_run_metric "SummaryRunMetricsOut.bin"
 
 
 Known Limitations

interop/constants/enums.h

@@ -14,7 +14,7 @@
 #include "interop/util/cstdint.h"
 
 /** Sentinel for an unknown enum type */
-#define INTEROP_UNKNOWN 0x400
+#define INTEROP_UNKNOWN 0x800
 
 
 /** Enumeration of specific features that can belong to a metric
@@ -30,6 +30,7 @@
         INTEROP_TUPLE_ASSIGN(BaseFeature, 0x08), \
         INTEROP_TUPLE_ASSIGN(ChannelFeature, 0x10), \
         INTEROP_TUPLE_ASSIGN(LaneFeature, 0x20), \
+        INTEROP_TUPLE_ASSIGN(DiskFeature, 0x400), \
         INTEROP_TUPLE1(UnknownMetricFeature)
 //NOTE: if we add any more features above, we should update the unknown metric value below
 
@@ -74,18 +75,19 @@
  * @see illumina::interop::constants::metric_group
  */
 #define INTEROP_ENUM_METRIC_GROUPS \
-        INTEROP_TUPLE2(CorrectedInt, CycleFeature|BaseFeature), \
-        INTEROP_TUPLE2(Error, CycleFeature), \
-        INTEROP_TUPLE2(Extraction, CycleFeature|ChannelFeature), \
-        INTEROP_TUPLE2(Image, CycleFeature|ChannelFeature), \
-        INTEROP_TUPLE2(Index, ReadFeature), \
-        INTEROP_TUPLE2(Q, CycleFeature), \
-        INTEROP_TUPLE2(Tile, TileFeature), \
+        INTEROP_TUPLE2(CorrectedInt, CycleFeature|BaseFeature|DiskFeature), \
+        INTEROP_TUPLE2(Error, CycleFeature|DiskFeature), \
+        INTEROP_TUPLE2(Extraction, CycleFeature|ChannelFeature|DiskFeature), \
+        INTEROP_TUPLE2(Image, CycleFeature|ChannelFeature|DiskFeature), \
+        INTEROP_TUPLE2(Index, ReadFeature|DiskFeature), \
+        INTEROP_TUPLE2(Q, CycleFeature|DiskFeature), \
+        INTEROP_TUPLE2(Tile, TileFeature|DiskFeature), \
         INTEROP_TUPLE2(QByLane, LaneFeature), \
         INTEROP_TUPLE2(QCollapsed, CycleFeature), \
-        INTEROP_TUPLE2(EmpiricalPhasing, CycleFeature), \
+        INTEROP_TUPLE2(EmpiricalPhasing, CycleFeature|DiskFeature), \
         INTEROP_TUPLE2(DynamicPhasing, CycleFeature), \
-        INTEROP_TUPLE2(ExtendedTile, TileFeature), \
+        INTEROP_TUPLE2(ExtendedTile, TileFeature|DiskFeature), \
+        INTEROP_TUPLE2(SummaryRun, DiskFeature), \
         INTEROP_TUPLE1(MetricCount),\
         INTEROP_TUPLE1(UnknownMetricGroup)
 
@@ -187,6 +189,8 @@
         INTEROP_TUPLE1(BaseReadType),\
         /** Lane base types are written out once for each lane and cycle */\
         INTEROP_TUPLE1(BaseLaneType),\
+        /** Run base types are written out once per run */\
+        INTEROP_TUPLE1(BaseRunType), \
         INTEROP_TUPLE1(BaseMetricCount),\
         INTEROP_TUPLE1(UnknownBaseType)
 

interop/constants/typedefs.h

@@ -22,6 +22,8 @@ namespace illumina { namespace interop { namespace constants
     typedef constant_type<metric_base_type, BaseReadType> base_read_t;
     /** Define base type for lane metrics */
     typedef constant_type<metric_base_type, BaseLaneType> base_lane_t;
+    /** Define base type for lane metrics */
+    typedef constant_type<metric_base_type, BaseRunType> base_run_t;
 
 }}}
 

interop/io/format/metric_format.h

@@ -288,7 +288,6 @@ namespace illumina { namespace interop { namespace io
                 else
                 {
                     const size_t offset = metric_offset_map[metric.id()];
-                    INTEROP_ASSERTMSG(metric_set[offset].lane() != 0, offset);
                     count += Layout::map_stream(in, metric_set[offset], metric_set, false);
                     INTEROP_ASSERT(metric_set[offset].id()>0);
                 }

interop/io/layout/base_metric.h

@@ -25,6 +25,53 @@ namespace illumina { namespace interop { namespace io { namespace layout
 {
 #pragma pack(1)
 
+    /** Empty class for InterOp records that contain run specific metrics
+     *
+     * These records contain both a lane and tile identifier.
+     *
+     * @note These classes are packed such that there is not padding. Their size reflects the accumulation of their
+     * member fields.
+     */
+    struct base_run_metric
+    {
+        /** Lane integral type */
+        typedef ::uint16_t lane_t;
+        /** Tile integral type */
+        typedef ::uint32_t tile_t;
+        /** Define a record size type */
+        typedef ::uint8_t record_size_t;
+        /** Define base type */
+        typedef constants::base_run_t base_t;
+
+        /** Constructor
+         */
+        base_run_metric() : m_dummy(0)
+        {
+        }
+
+        /** Set the lane and tile id from a base metric
+         *
+         * @param metric a base_metric from the model
+         */
+        template<class BaseMetric>
+        void set(const BaseMetric &/*metric*/)
+        {
+            m_dummy = 0;
+        }
+
+        /** Test if the layout contains valid data
+         *
+         * @return true if data is valid
+         */
+        bool is_valid() const
+        {
+            return true;
+        }
+
+    private:
+        lane_t m_dummy;
+    };
+
     /** Base class for InterOp records that contain tile specific metrics
      *
      * These records contain both a lane and tile identifier.

interop/io/metric_file_stream.h

@@ -41,14 +41,14 @@ namespace illumina { namespace interop { namespace io
      * @return number of bytes written
      */
     template<class MetricSet>
-    size_t write_interop_to_buffer(const MetricSet& metrics, ::uint8_t* buffer, size_t buffer_size)
+    size_t write_interop_to_buffer(const MetricSet& metrics, ::uint8_t* buffer, const size_t buffer_size)
                         INTEROP_THROW_SPEC((io::invalid_argument, io::bad_format_exception, io::incomplete_file_exception, io::format_exception))
     {
         std::ostringstream fout;
         write_metrics(fout, metrics, metrics.version());
         std::string str = fout.str();
         if(buffer_size < str.length())
-            INTEROP_THROW(invalid_argument, "Buffer size too small");
+            INTEROP_THROW(invalid_argument, "Buffer size too small: " << buffer_size << " < " << str.length());
         size_t i=0;
         for(;i<str.length();++i)
             buffer[i] = static_cast< ::uint8_t >(str[i]);
@@ -63,7 +63,7 @@ namespace illumina { namespace interop { namespace io
      * @throw incomplete_file_exception
      */
     template<class MetricSet>
-    void read_interop_from_buffer(::uint8_t* buffer, size_t buffer_size, MetricSet& metrics)  INTEROP_THROW_SPEC(
+    void read_interop_from_buffer(::uint8_t* buffer, const size_t buffer_size, MetricSet& metrics)  INTEROP_THROW_SPEC(
                                                                             (interop::io::file_not_found_exception,
                                                                             interop::io::bad_format_exception,
                                                                             interop::io::incomplete_file_exception,

interop/logic/summary/summary_statistics.h

@@ -148,6 +148,7 @@ namespace illumina { namespace interop { namespace logic { namespace summary
     template<typename I, typename S>
     void summarize(I beg, I end, S &stat, const bool skip_median)
     {
+        stat.clear();
         if (beg == end) return;
         stat.mean(util::mean<float>(beg, end));
         stat.stddev(std::sqrt(util::variance_with_mean<float>(beg, end, stat.mean())));
@@ -196,6 +197,31 @@ namespace illumina { namespace interop { namespace logic { namespace summary
         return size_t(std::distance(beg, end));
     }
 
+    /** Calculate the sum over a collection of values, ignoring NaNs
+     *
+     * @param beg iterator to start of collection
+     * @param end iterator to end of collection
+     * @param init initial value for accumulate call
+     * @param op unary/binary operator for getting a value in a complex object
+     * @return sum of init + applying op to the range [beg, end)
+     */
+    template<typename I, typename S, typename Op>
+    S nan_accumulate(I beg, I end, const S init, Op op)
+    {
+        S return_value = init;
+        if (beg == end) return init;
+        const float temp_value = 0;
+        for (; beg != end; ++beg)
+        {
+            const float current_value = op(temp_value, *beg);
+            if(!std::isnan(current_value))
+            {
+                return_value += static_cast<S>(current_value);
+            }
+        }
+        return return_value;
+    }
+
     /** Safe divide
      *
      * @param num numerator

interop/logic/summary/tile_summary.h

@@ -89,16 +89,16 @@ namespace illumina { namespace interop { namespace logic { namespace summary
                   util::op::const_member_function_less(&model::metrics::tile_metric::percent_pf),
                   skip_median);
         stat_summary.percent_pf(stat);
-        stat_summary.reads(std::accumulate(tile_data.begin(),
+        stat_summary.reads(nan_accumulate(tile_data.begin(),
                                            tile_data.end(),
-                                           float(0),
+                                          float(0),
                                            util::op::const_member_function(
                                                    &model::metrics::tile_metric::cluster_count)));
-        stat_summary.reads_pf(std::accumulate(tile_data.begin(),
-                                              tile_data.end(),
-                                              float(0),
-                                              util::op::const_member_function(
-                                                      &model::metrics::tile_metric::cluster_count_pf)));
+        stat_summary.reads_pf(nan_accumulate(tile_data.begin(),
+                                             tile_data.end(),
+                                             float(0),
+                                             util::op::const_member_function(
+                                                     &model::metrics::tile_metric::cluster_count_pf)));
     }
     /** Update the stat summary with cached read metrics
      *

interop/model/metric_base/base_metric.h

@@ -25,6 +25,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base
 
     // Forward declaration
     class base_metric;
+    class empty_metric;
 
     /** Get attributes of the metric */
     template<class Metric>
@@ -58,7 +59,7 @@ namespace illumina { namespace interop { namespace model { namespace metric_base
          *
          * @todo remove this method
          */
-        void update_max_cycle(const base_metric &)
+        void update_max_cycle(const empty_metric &)
         { }
     };
 
@@ -68,6 +69,58 @@ namespace illumina { namespace interop { namespace model { namespace metric_base
      */
     class empty_metric
     {
+    public:
+        /** id_t type */
+        typedef ::uint32_t id_t;
+        /** Unsigned int
+         */
+        typedef ::uint32_t uint_t;
+        /** Define the base type */
+        typedef constants::base_run_t base_t;
+
+    public:
+        /** Set the base metric identifiers
+         *
+         * @param base layout base
+         */
+        template<class BaseMetric>
+        void set_base(const BaseMetric &/*base*/)
+        {
+        }
+        /** Set id
+         *
+         * @param lane lane number
+         * @param tile tile number
+         */
+        void set_base(const uint_t /*lane*/, const uint_t /*tile*/)
+        {
+        }
+        /** Get the metric name suffix
+         *
+         * @return empty string
+         */
+        static const char *suffix()
+        {
+            return "";
+        }
+
+        /** Comparison operator used to sort the entries in order of their IDs
+         *
+         * @param metric2 metric to compare with the current object
+         * @return true if this object's ID is less than metric2's ID
+         */
+        bool operator< (const empty_metric& /*metric2*/) const
+        {
+            return false;
+        }
+        /** Unique id created from both the lane and tile
+         *
+         * @return 1
+         */
+        static id_t create_id(const id_t, const id_t, const id_t= 0)// TODO: remove hack (const id_t=0)
+        {
+            return 1;// Cannot be zero
+        }
     };
 
     /** Base class for InterOp classes that contain tile specific metrics