From 367e44acd5c399d5e315b24fb32683f6c5afe3ec Mon Sep 17 00:00:00 2001 From: Robert Langlois Date: Thu, 18 Aug 2022 11:57:18 -0700 Subject: [PATCH] Add support for error metrics v6 (#296) * Support future platforms using error metrics v6 * Fix version history * Fix windows python build * Fix testing branches on same day --- .github/workflows/python.yml | 1 + docs/src/changes.md | 2 + interop/io/metric_file_stream.h | 39 +++ .../metrics/corrected_intensity_metric.h | 17 +- interop/model/metrics/error_metric.h | 176 +++++++++++-- src/ext/python/core.py | 1 - src/interop/model/metrics/error_metric.cpp | 245 +++++++++++++++++- .../model/metrics/extended_tile_metric.cpp | 1 - .../model/metrics/q_collapsed_metric.cpp | 10 +- src/interop/model/metrics/q_metric.cpp | 12 +- .../model/metrics/summary_run_metric.cpp | 3 + src/tests/csharp/metrics/ErrorMetricsTest.cs | 2 +- .../interop/metrics/error_metrics_test.cpp | 43 ++- .../interop/metrics/inc/error_metrics_test.h | 46 ++++ .../metrics/inc/metric_format_fixtures.h | 1 + .../interop/metrics/metric_streams_test.cpp | 21 +- 16 files changed, 570 insertions(+), 50 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a0832d5a5..07cbcabe3 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -79,6 +79,7 @@ jobs: password: ${{ secrets.TEST_PYPI_API_TOKEN }} packages_dir: ./dist repository_url: https://test.pypi.org/legacy/ + skip_existing: true publish: needs: [build] diff --git a/docs/src/changes.md b/docs/src/changes.md index 3efa55c9a..5cc16340f 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -5,6 +5,8 @@ Date | Description ---------- | ----------- +2022-08-09 | Support parsing error metrics v6 +2022-08-09 | Fix potential bugs when parsing Q-metrics 2022-08-13 | Issue-282: Support universal build for M1 mac 2022-08-13 | Issue-282: Remove support for Python 3.5 (not working on Mac M1) 2022-08-13 | Issue-282: Update Windows Agent to 2019 from 2016 (2016 discontinued) diff --git a/interop/io/metric_file_stream.h b/interop/io/metric_file_stream.h index 459f5d3b0..c6b1d0a64 100644 --- a/interop/io/metric_file_stream.h +++ b/interop/io/metric_file_stream.h @@ -91,6 +91,26 @@ namespace illumina { namespace interop { namespace io std::istringstream in(buffer); read_metrics(in, metrics, buffer.length(), rebuild); } + /** Write the binary InterOp file into the given string using the given metric set + * + * @param buffer string holding a byte buffer + * @param metrics metric set + * @param version version of the format to write (-1 means use latest) + * @throw bad_format_exception + * @throw incomplete_file_exception + * @throw model::index_out_of_bounds_exception + */ + template + void write_interop_to_string(std::string& buffer + , const MetricSet& metrics, const ::int16_t version = -1) INTEROP_THROW_SPEC( + (interop::io::bad_format_exception, + interop::io::incomplete_file_exception, + model::index_out_of_bounds_exception)) + { + std::ostringstream out; + write_metrics(out, metrics, version); + buffer = out.str(); + } /** Read the binary InterOp file into the given metric set * * @param buffer string holding a byte buffer @@ -108,6 +128,25 @@ namespace illumina { namespace interop { namespace io std::istringstream in(buffer); return read_header(in, metrics); } + /** Write the binary InterOp file header into a string from the given metric set + * + * @param buffer string holding a byte buffer + * @param metrics metric set + * @throw bad_format_exception + * @throw incomplete_file_exception + * @throw model::index_out_of_bounds_exception + */ + template + void write_header_to_string(std::string& buffer, const MetricSet& metrics, ::int16_t version=-1) INTEROP_THROW_SPEC( + (interop::io::bad_format_exception, + interop::io::incomplete_file_exception, + model::index_out_of_bounds_exception)) + { + if(version == -1) version = metrics.version(); + std::ostringstream out; + write_metric_header(out, version, metrics); + buffer = out.str(); + } /** Read the binary InterOp file into the given metric set * * @snippet src/examples/example1.cpp Reading a binary InterOp file diff --git a/interop/model/metrics/corrected_intensity_metric.h b/interop/model/metrics/corrected_intensity_metric.h index 9d88a881c..12e9bce97 100644 --- a/interop/model/metrics/corrected_intensity_metric.h +++ b/interop/model/metrics/corrected_intensity_metric.h @@ -220,7 +220,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param lane lane number * @param tile tile number * @param cycle cycle number - * @param called_count_vec number of clusters called per base + * @param called_count number of clusters called per base */ corrected_intensity_metric(const uint_t lane, const uint_t tile, @@ -434,10 +434,10 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ float percent_base(const constants::dna_bases index) const INTEROP_THROW_SPEC((model::index_out_of_bounds_exception)) { - uint_t total = total_calls(index == constants::NC); + const uint_t total = total_calls(index == constants::NC ); if (total == 0) return std::numeric_limits::quiet_NaN(); - return called_counts(index) / static_cast(total) * 100; + return called_counts(index) / static_cast(total) * 100.0f; } /** Get the percentage per base (does not include no calls) @@ -447,11 +447,14 @@ namespace illumina { namespace interop { namespace model { namespace metrics */ float_array_t percent_bases() const { - uint_t total = total_calls(); - float_array_t percent_bases(called_counts_array().size() - 1); + if(m_called_counts.empty()) + { + return float_array_t(); + } + const uint_t total = total_calls(); + float_array_t percent_bases(constants::NUM_OF_BASES); for (size_t i = 0; i < percent_bases.size(); ++i) - percent_bases[i] = (total == 0) ? std::numeric_limits::quiet_NaN() : - called_counts_array()[i + 1] / static_cast(total) * 100; + percent_bases[i] = static_cast(m_called_counts[i + 1]) / static_cast(total) * 100.0f; return percent_bases; } diff --git a/interop/model/metrics/error_metric.h b/interop/model/metrics/error_metric.h index 95dc93d5a..8af323d8e 100644 --- a/interop/model/metrics/error_metric.h +++ b/interop/model/metrics/error_metric.h @@ -24,12 +24,90 @@ namespace illumina { namespace interop { namespace model { namespace metrics { + /** Header information for an error metric file, used for format version 6 + */ + class error_metric_header : public metric_base::base_cycle_metric::header_type + { + public: + /** Constructor + */ + error_metric_header(const std::vector& adapters) : + m_number_adapters(static_cast(adapters.size())), + m_adapter_length(0) + { + if(adapters.size()>0) + m_adapter_length = static_cast(adapters[0].size()); + for(size_t i = 0; i < adapters.size(); i++) { + INTEROP_ASSERTMSG(adapters[i].size() == m_adapter_length, + "Adapter Sequence (" << adapters[i] << ") length doesn't match expected adapter length"); + m_adapter_sequences.push_back(adapters[i]); + } + } + + error_metric_header() : + m_number_adapters(0), + m_adapter_length(0) + {} + + /** number of adapter sequences in model + * + * @return m_number_adapters + */ + uint16_t number_adapters() const + { + return m_number_adapters; + } + + /** Length of each adapter sequence + * + * @return m_adapter_length + */ + uint16_t adapter_length() const + { + return m_adapter_length; + } + + /** Adapter sequences + * + * @return m_adapter_sequences + */ + const std::vector& adapter_sequences() const + { + return m_adapter_sequences; + } + + void clear() + { + m_number_adapters = 0; + m_adapter_length = 0; + m_adapter_sequences.clear(); + metric_base::base_cycle_metric::header_type::clear(); + } + /** Generate a default header + * + * @return default header + */ + static error_metric_header default_header() + { + return error_metric_header(std::vector()); + } + + private: + uint16_t m_number_adapters; + uint16_t m_adapter_length; + std::vector m_adapter_sequences; + + template + friend + struct io::generic_layout; + }; + /** @brief Error rate for a spiked in PhiX control sample * * The error metric is the calculated error rate, as determined by a spiked in PhiX control sample. * This metric is available for each lane and tile for every cycle. * - * @note Supported versions: 3, 4, and 5 + * @note Supported versions: 3, 4, 5 and 6 */ class error_metric : public metric_base::base_cycle_metric { @@ -40,29 +118,35 @@ namespace illumina { namespace interop { namespace model { namespace metrics MAX_MISMATCH = 5, /** Unique type code for metric */ TYPE = constants::Error, - /** Latest version of the InterOp format */ - LATEST_VERSION = 5 + /** Latest version of the I + * nterOp format */ + LATEST_VERSION = 6 }; /** Define a uint array using an underlying vector */ typedef std::vector uint_array_t; + /** Vector of floats points */ + typedef std::vector float_vector; + /** Error metric set header */ + typedef error_metric_header header_type; public: /** Constructor */ error_metric() : - metric_base::base_cycle_metric(0, 0, 0), - m_error_rate(std::numeric_limits::quiet_NaN()), - m_phix_adapter_rate(std::numeric_limits::quiet_NaN()), - m_mismatch_cluster_count(MAX_MISMATCH, 0) + metric_base::base_cycle_metric(0, 0, 0), + m_error_rate(std::numeric_limits::quiet_NaN()), + m_phix_adapter_rate(std::numeric_limits::quiet_NaN()), + m_mismatch_cluster_count(MAX_MISMATCH, 0) { } /** Constructor */ - error_metric(const header_type&) : - metric_base::base_cycle_metric(0, 0, 0), - m_error_rate(std::numeric_limits::quiet_NaN()), - m_phix_adapter_rate(std::numeric_limits::quiet_NaN()), - m_mismatch_cluster_count(MAX_MISMATCH, 0) + error_metric(const header_type& header) : + metric_base::base_cycle_metric(0, 0, 0), + m_error_rate(std::numeric_limits::quiet_NaN()), + m_phix_adapter_rate(std::numeric_limits::quiet_NaN()), + m_phix_adapter_rates(header.number_adapters(), 0), + m_mismatch_cluster_count(MAX_MISMATCH, 0) { } @@ -78,13 +162,34 @@ namespace illumina { namespace interop { namespace model { namespace metrics const uint_t cycle, const float error, const float phix_adapter_rate) : - metric_base::base_cycle_metric(lane, tile, cycle), - m_error_rate(error), - m_phix_adapter_rate(phix_adapter_rate), - m_mismatch_cluster_count(MAX_MISMATCH, 0) + metric_base::base_cycle_metric(lane, tile, cycle), + m_error_rate(error), + m_phix_adapter_rate(phix_adapter_rate), + m_mismatch_cluster_count(MAX_MISMATCH, 0) { } + /** Constructor + * + * @param lane lane number + * @param tile tile number + * @param cycle cycle number + * @param error error rate for current cycle + * @param phix_adapter_rates rates of each adapter + */ + error_metric(const uint_t lane, + const uint_t tile, + const uint_t cycle, + const float error, + const float_vector &phix_adapter_rates) : + metric_base::base_cycle_metric(lane, tile, cycle), + m_error_rate(error), + m_phix_adapter_rates(phix_adapter_rates), + m_mismatch_cluster_count(MAX_MISMATCH, 0) + { + set_mean_adapter_rate(); + } + public: /** @defgroup error_metric Error Metrics * @@ -114,6 +219,34 @@ namespace illumina { namespace interop { namespace model { namespace metrics { return m_phix_adapter_rate; } + /* Calculated adapter trim rate per adapter of PhiX clusters + * + * @note Supported by v6 + * @return vector of adapter rates per adapter + */ + const float_vector& phix_adapter_rates() const + { + return m_phix_adapter_rates; + } + /* Calculated adapter trim rate per adapter of PhiX clusters + * + * @note Supported by v6 + * @return adapter rate for given adapter index + */ + float phix_adapter_rate_at(const size_t n) const INTEROP_THROW_SPEC((model::index_out_of_bounds_exception)) + { + INTEROP_BOUNDS_CHECK(n, m_phix_adapter_rates.size(), "Index out of bounds"); + return m_phix_adapter_rates[n]; + } + /* Number of adapters on PhiX + * + * @note Supported by v6 + * @return number of adapters + */ + size_t phix_adapter_count() const + { + return m_phix_adapter_rates.size(); + } /** Number of clusters at given number of mismatches * @@ -175,8 +308,19 @@ namespace illumina { namespace interop { namespace model { namespace metrics { return "Error"; } private: + + void set_mean_adapter_rate() { + const size_t size = m_phix_adapter_rates.size(); + if(size == 0) return; + m_phix_adapter_rate = 0; + for(size_t i = 0; i < size; i++) + m_phix_adapter_rate += m_phix_adapter_rates[i]; + m_phix_adapter_rate /= size; + } + float m_error_rate; float m_phix_adapter_rate; + float_vector m_phix_adapter_rates; uint_array_t m_mismatch_cluster_count; template friend diff --git a/src/ext/python/core.py b/src/ext/python/core.py index 7ff2ed5f0..2a7f9d21d 100644 --- a/src/ext/python/core.py +++ b/src/ext/python/core.py @@ -689,7 +689,6 @@ def imaging(run_metrics, dtype='f4', **extra): if not isinstance(dtype, str): dtype = np.dtype(dtype).str - return np.core.records.fromarrays(data.transpose() , names=",".join(headers) , formats=",".join([dtype for _ in headers])) diff --git a/src/interop/model/metrics/error_metric.cpp b/src/interop/model/metrics/error_metric.cpp index 96506fe1d..25c5f220e 100644 --- a/src/interop/model/metrics/error_metric.cpp +++ b/src/interop/model/metrics/error_metric.cpp @@ -249,8 +249,8 @@ namespace illumina{ namespace interop{ namespace io static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) { std::streamsize count = 0; - count += stream_map< error_t >(stream, metric.m_error_rate); - count += stream_map< error_t >(stream, metric.m_phix_adapter_rate); + count += stream_map< error_t >(stream, metric.m_error_rate); // 4 + count += stream_map< error_t >(stream, metric.m_phix_adapter_rate); // 4 return count; } /** Compute the layout size @@ -259,9 +259,9 @@ namespace illumina{ namespace interop{ namespace io */ static record_size_t compute_size(const error_metric::header_type&) { - return static_cast(sizeof(metric_id_t)+ - sizeof(error_t)+ // m_errorRate - sizeof(error_t) // m_phix_adapter_rate + return static_cast(sizeof(metric_id_t)+ // 2 + 2 + 4 + sizeof(error_t)+ // m_errorRate 4 + sizeof(error_t) // m_phix_adapter_rate 4 ); } /** Compute header size @@ -274,6 +274,168 @@ namespace illumina{ namespace interop{ namespace io } }; + + /** Error Metric Record Layout Version 6 + * + * This class provides an interface to reading the error metric file: + * - InterOp/ErrorMetrics.bin + * - InterOp/ErrorMetricsOut.bin + * + * The class takes two template arguments: + * + * 1. Metric Type: error_metric + * 2. Version: 6 + */ + template<> + struct generic_layout : public default_layout<6> + { + /** @page error_v6 Error Version 6 + * + * This class provides an interface to reading the error metric file: + * - InterOp/ErrorMetrics.bin + * - InterOp/ErrorMetricsOut.bin + * + * The file format for error metrics is as follows: + * + * @b Header + * + * illumina::interop::io::read_metrics (Function that parses this information) + * + * byte 0: version number (6) + * byte 1: record size (12+4*num_adapter) + * + * @b Extended Header + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * byte 2-3: num_adapter (uint16) + * byte 4-5: adapter_base_cnt (uint16) + * byte 6-(6+num_adapter*adapter_base_cnt) (uint8) + * + * + * @b n-Records + * + * illumina::interop::io::layout::base_cycle_metric (Class that parses this information) + * + * 2 bytes: lane number (uint16) + * 4 bytes: tile number (uint32) + * 2 bytes: cycle number (uint16) + * + * illumina::interop::io::generic_layout (Class that parses this information) + * + * 4 bytes: error rate (float32) + * 4*num_adapter bytes: vector of fraction of reads adapter-trimmed at the cycle (float32) + */ + /** Metric ID type */ + typedef layout::base_cycle_metric< ::uint32_t > metric_id_t; + /** Error type */ + typedef float error_t; + /** Number of adapters type */ + typedef uint16_t num_adapters_t; + /** Adapter length type */ + typedef uint16_t adapter_length_t; + /** Map reading/writing to stream + * + * Reading and writing are symmetric operations, map it once + * + * @param stream input/output stream + * @param metric source/destination metric + * @return number of bytes read or total number of bytes written + */ + template + static std::streamsize map_stream(Stream& stream, Metric& metric, Header&, const bool) + { + std::streamsize count = 0; + count += stream_map< error_t >(stream, metric.m_error_rate); + count += stream_map< error_t >(stream, metric.m_phix_adapter_rates, + metric.m_phix_adapter_rates.size()); + update(metric); + return count; + } + template + static void update(const Metric& /*metric*/) + { + } + template + static void update(Metric& metric) + { + metric.set_mean_adapter_rate(); + } + + /** Read metric from the input stream + * + * @param in input stream + * @param header destination header + * @return sentinel + */ + template + static std::streamsize map_stream_for_header(std::istream &in, Header &header)// TODO clean this up + { + std::streamsize count = 0; + count += stream_map(in, header.m_number_adapters); + if (in.fail()) return count; + count += stream_map(in, header.m_adapter_length); + if (in.fail()) return count; + map_resize(header.m_adapter_sequences, header.m_number_adapters); + for(size_t i = 0; i < header.number_adapters(); i++){ + std::string seq; + char c; + for(size_t j = 0; j < header.adapter_length(); j++){ + count += stream_map(in, c); + seq += c; + } + header.m_adapter_sequences[i] = seq; + } + return count; + } + + /** Write metric to the output stream + * + * @param out output stream + * @param header source header + * @return sentinel + */ + template + static std::streamsize map_stream_for_header(std::ostream &out, Header &header) + { + std::streamsize count = 0; + count += stream_map(out, header.m_number_adapters); + count += stream_map(out, header.m_adapter_length); + for(size_t i = 0; i < header.number_adapters(); i++){ + std::string seq = header.m_adapter_sequences[i]; + for(size_t j = 0; j < header.adapter_length(); j++){ + count += stream_map(out, seq[i]); + } + } + return count; + } + + /** Compute the layout size + * + * @return size of the record + */ + static record_size_t compute_size(const error_metric::header_type& header) + { + return static_cast( + sizeof(metric_id_t)+ // 2 + 2 + 4 + sizeof(error_t)+ // m_errorRate 4 + sizeof(error_t)*header.number_adapters() // m_phix_adapter_rates 4*num_adapters + ); + } + /** Compute header size + * + * @return header size + */ + static record_size_t compute_header_size(const error_metric::header_type& header) + { + return static_cast( + sizeof(record_size_t) + sizeof(version_t) + + sizeof(num_adapters_t) + sizeof(adapter_length_t) + + sizeof(char)*header.m_adapter_length*header.m_number_adapters + ); + } + + }; #pragma pack() /** Error Metric CSV text format * @@ -365,6 +527,70 @@ namespace illumina{ namespace interop{ namespace io out << sep << headers[i]; out << eol; return util::length_of(headers); + } + /** Write a error metric to the output stream + * + * @param out output stream + * @param metric error metric + * @param sep column separator + * @param eol row separator + * @return number of columns written + */ + static size_t write_metric(std::ostream& out, + const error_metric& metric, + const header_type&, + const char sep, + const char eol, + const char) + { + out << metric.lane() << sep << metric.tile() << sep << metric.cycle() << sep; + out << metric.error_rate() << sep << metric.phix_adapter_rate() << eol; + return 0; + } + }; + + + /** Error Metric CSV text format + * + * This class provide an interface for writing the error metrics to a CSV file: + * + * - ErrorMetrics.csv + */ + template<> + struct text_layout< error_metric, 3 > + { + /** Define a header type */ + typedef error_metric::header_type header_type; + /** Write header to the output stream + * + * @param out output stream + * @param sep column separator + * @param eol row separator + * @return number of column headers + */ + static size_t write_header(std::ostream& out, + const header_type& header, + const std::vector&, + const char sep, + const char eol) + { + const char* column_headers[] = + { + "Lane", "Tile", "Cycle", "ErrorRate" + }; + std::vector headers; + headers.reserve(util::length_of(column_headers)+header.number_adapters()); + for(size_t i=0;i(header.number_adapters());++i) + headers.push_back(std::string()+"PhiXAdapterRate("+std::to_string(i)+")"); + + out << "# Column Count: " << util::length_of(headers) << eol; + out << headers[0]; + for(size_t i=1;i(header.number_adapters());++i) + out << sep << metric.phix_adapter_rates()[i]; + out << eol; return 0; } }; @@ -392,8 +621,10 @@ INTEROP_FORCE_LINK_DEF(error_metric) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(error_metric, 3 ) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(error_metric, 4 ) INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(error_metric, 5 ) +INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(error_metric, 6 ) // Text formats INTEROP_REGISTER_METRIC_TEXT_LAYOUT(error_metric, 1) INTEROP_REGISTER_METRIC_TEXT_LAYOUT(error_metric, 2) +INTEROP_REGISTER_METRIC_TEXT_LAYOUT(error_metric, 3) \ No newline at end of file diff --git a/src/interop/model/metrics/extended_tile_metric.cpp b/src/interop/model/metrics/extended_tile_metric.cpp index 92aca818c..afddabab8 100644 --- a/src/interop/model/metrics/extended_tile_metric.cpp +++ b/src/interop/model/metrics/extended_tile_metric.cpp @@ -238,7 +238,6 @@ namespace illumina{ namespace interop{ namespace io struct generic_layout : public default_layout<3> { /** @page extended_tile_v3 Extended Tile Version 3 - * * * This class provides an interface to reading the extended tile metric file: * - InterOp/ExtendedTileMetrics.bin diff --git a/src/interop/model/metrics/q_collapsed_metric.cpp b/src/interop/model/metrics/q_collapsed_metric.cpp index 0c270196f..12cca4a6e 100644 --- a/src/interop/model/metrics/q_collapsed_metric.cpp +++ b/src/interop/model/metrics/q_collapsed_metric.cpp @@ -373,7 +373,7 @@ namespace illumina{ namespace interop{ namespace io { /** Define the size of the record with the ID */ TOTAL_RECORD_SIZE = RECORD_SIZE+sizeof(metric_id_t), /** Alternative record size*/ - ALT_RECORD_SIZE=TOTAL_RECORD_SIZE+sizeof(median_t) + ALT_RECORD_SIZE=TOTAL_RECORD_SIZE+sizeof(median_t) //22 }; /** Map reading/writing to stream @@ -460,15 +460,17 @@ namespace illumina{ namespace interop{ namespace io { * * @return header size */ - static record_size_t compute_header_size(const q_collapsed_metric::header_type& header) + static record_size_t compute_header_size(const q_collapsed_metric::header_type& /*header*/) { - if (header.bin_count() == 0) + return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); + // Never write the header + /*if (header.bin_count() == 0) return static_cast(sizeof(record_size_t) + sizeof(version_t) + sizeof(bool_t)); return static_cast(sizeof(record_size_t) + sizeof(version_t) + // version sizeof(bool_t) + // has bins sizeof(bin_count_t) + // number of bins - header.bin_count() * 3 * sizeof(bin_t)); + header.bin_count() * 3 * sizeof(bin_t));*/ } /** Does not read/write record size, this is done in `map_stream_for_header` * diff --git a/src/interop/model/metrics/q_metric.cpp b/src/interop/model/metrics/q_metric.cpp index add3faf05..1d07e8d1e 100644 --- a/src/interop/model/metrics/q_metric.cpp +++ b/src/interop/model/metrics/q_metric.cpp @@ -278,6 +278,7 @@ namespace illumina { namespace interop { namespace io copy_value_write(header.m_qscore_bins, tmp); count += stream_map(stream, tmp, bin_count); + if(stream.fail()) return count; copy_value_read(header.m_qscore_bins, tmp); return count; @@ -346,7 +347,10 @@ namespace illumina { namespace interop { namespace io static void copy_value_read(std::vector &bins, bin_t (&tmp)[N]) { INTEROP_ASSERT(bins.size() <= N); - for (size_t i = 0; i < bins.size(); ++i) bins[i].m_value = tmp[i]; + for (size_t i = 0; i < bins.size(); ++i) + { + bins[i].m_value = tmp[i]; + } } static void copy_value_read(const std::vector &, bin_t *) @@ -492,6 +496,7 @@ namespace illumina { namespace interop { namespace io copy_value_write(header.m_qscore_bins, tmp); count += stream_map(stream, tmp, bin_count); + if(stream.fail()) return count; copy_value_read(header.m_qscore_bins, tmp); return count; @@ -560,7 +565,10 @@ namespace illumina { namespace interop { namespace io static void copy_value_read(std::vector &bins, bin_t (&tmp)[N]) { INTEROP_ASSERT(bins.size() <= N); - for (size_t i = 0; i < bins.size(); ++i) bins[i].m_value = tmp[i]; + for (size_t i = 0; i < bins.size(); ++i) + { + bins[i].m_value = tmp[i]; + } } static void copy_value_read(const std::vector &, bin_t *) diff --git a/src/interop/model/metrics/summary_run_metric.cpp b/src/interop/model/metrics/summary_run_metric.cpp index 6577bfbab..7d9ae9046 100644 --- a/src/interop/model/metrics/summary_run_metric.cpp +++ b/src/interop/model/metrics/summary_run_metric.cpp @@ -38,6 +38,7 @@ namespace illumina { namespace interop { namespace io struct generic_layout : public default_layout<1> { /** @page summary_run_v1 SummaryRun Version 1 + * * * This class provides an interface to reading the SummaryRun metric file: * - InterOp/SummaryRun.bin @@ -190,3 +191,5 @@ INTEROP_REGISTER_METRIC_GENERIC_LAYOUT(summary_run_metric, 1) // Text formats INTEROP_REGISTER_METRIC_TEXT_LAYOUT(summary_run_metric, 1) + + diff --git a/src/tests/csharp/metrics/ErrorMetricsTest.cs b/src/tests/csharp/metrics/ErrorMetricsTest.cs index 37cb6a5e3..2844923ea 100644 --- a/src/tests/csharp/metrics/ErrorMetricsTest.cs +++ b/src/tests/csharp/metrics/ErrorMetricsTest.cs @@ -23,7 +23,7 @@ public class ErrorMetricsTestV3 [SetUp] protected void SetUp() { - base_cycle_metric_header header = new base_cycle_metric_header(); + error_metric_header header = new error_metric_header(); expected_metrics.Add(new error_metric(7, 1114, 1, 0.450100899f, 0f)); expected_metrics.Add(new error_metric(7, 1114, 2, 0.900201797f, 0f)); expected_metrics.Add(new error_metric(7, 1114, 3, 0.465621591f, 0f)); diff --git a/src/tests/interop/metrics/error_metrics_test.cpp b/src/tests/interop/metrics/error_metrics_test.cpp index fc416978b..f418cf578 100644 --- a/src/tests/interop/metrics/error_metrics_test.cpp +++ b/src/tests/interop/metrics/error_metrics_test.cpp @@ -31,15 +31,19 @@ error_metrics_tests::generator_type error_unit_test_generators[] = { wrap(new hardcoded_metric_generator< error_metric_v3 >) , wrap(new hardcoded_metric_generator< error_metric_v4 >), wrap(new hardcoded_metric_generator< error_metric_v5 >), + wrap(new hardcoded_metric_generator< error_metric_v6 >), wrap(new write_read_metric_generator< error_metric_v3 >), wrap(new write_read_metric_generator< error_metric_v4 >), wrap(new write_read_metric_generator< error_metric_v5 >), + wrap(new write_read_metric_generator< error_metric_v6 >), wrap(new by_cycle_metric_generator< error_metric_v3 >), wrap(new by_cycle_metric_generator< error_metric_v4 >), wrap(new by_cycle_metric_generator< error_metric_v5 >), + wrap(new by_cycle_metric_generator< error_metric_v6 >), wrap(new clear_metric_generator< error_metric_v3 >), wrap(new clear_metric_generator< error_metric_v4 >), - wrap(new clear_metric_generator< error_metric_v5 >) + wrap(new clear_metric_generator< error_metric_v5 >), + wrap(new clear_metric_generator< error_metric_v6 >) }; // Setup unit tests for error_metrics_tests @@ -72,6 +76,10 @@ TEST_P(error_metrics_tests, compare_expected_actual) EXPECT_EQ(it_expected->mismatch_count(), it_actual->mismatch_count()); EXPECT_NEAR(it_expected->error_rate(), it_actual->error_rate(), 1e-5f); INTEROP_EXPECT_NEAR(it_expected->phix_adapter_rate(), it_actual->phix_adapter_rate(), 1e-5f); + ASSERT_EQ(it_expected->phix_adapter_count(), it_actual->phix_adapter_count()); + for(size_t i=0;iphix_adapter_count();++i) + INTEROP_EXPECT_NEAR(it_expected->phix_adapter_rate_at(i), it_actual->phix_adapter_rate_at(i), 1e-5f); + for(ptrdiff_t i=0;i(it_expected->mismatch_count());i++) EXPECT_EQ(it_expected->mismatch_cluster_count(i), it_actual->mismatch_cluster_count(i)); } @@ -88,6 +96,39 @@ TEST(error_metrics_single_test, test_tile_metric_count_for_lane) } +/** + * @test Ensure the adapter sequences are correct in header + */ +TEST(error_metrics_single_test, test_error_header_v6) +{ + error_metric_set metrics; + error_metric_v6::create_expected(metrics); + EXPECT_EQ(metrics.number_adapters(), 2u); + EXPECT_EQ(metrics.adapter_length(), 8u); + EXPECT_EQ(metrics.adapter_sequences().size(), 2u); + EXPECT_EQ(metrics.adapter_sequences()[0], "CCCCCCCC"); + EXPECT_EQ(metrics.adapter_sequences()[1], "AAAAAAAA"); +} + +/** + * @test Ensure the rates per sequences are correct + */ +TEST(error_metrics_single_test, test_error_rates_v6) +{ + error_metric_set metrics; + error_metric_v6::create_expected(metrics); + + float rates[3][2] = {{0.298748f, 0.1234f}, + {0.287257f, 0.3934f}, + {0.608985f, 0.5034f}}; + + EXPECT_EQ(metrics.size(), 3u); + size_t i = 0; + for(error_metric_v6::const_iterator it=metrics.begin(); it != metrics.end();it++, i++) { + EXPECT_EQ(it->phix_adapter_rate(), (rates[i][0]+rates[i][1])/2); + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test diff --git a/src/tests/interop/metrics/inc/error_metrics_test.h b/src/tests/interop/metrics/inc/error_metrics_test.h index 14fc34fd1..cdadfef32 100644 --- a/src/tests/interop/metrics/inc/error_metrics_test.h +++ b/src/tests/interop/metrics/inc/error_metrics_test.h @@ -364,5 +364,51 @@ namespace illumina { namespace interop { namespace unittest buffer.assign(tmp, tmp+util::length_of(tmp)); } }; + + /** This test writes three records of an InterOp files, then reads them back in and compares + * each value to ensure they did not change. + * + * @see model::metrics::error_metric + * @note Version 6 + */ + struct error_metric_v6 : metric_test + { + /** Create the expected metric set + * + * @param metrics destination metric set + */ + static void create_expected(metric_set_t &metrics, const model::run::info& =model::run::info()) + { + std::vector adapters; + adapters.push_back("CCCCCCCC"); + adapters.push_back("AAAAAAAA"); + header_t header = header_t(adapters); + metrics = metric_set_t(header, VERSION); + + float rates1[2] = {0.298748f, 0.1234f}; + float rates2[2] = {0.287257f, 0.3934f}; + float rates3[2] = {0.608985f, 0.5034f}; + metrics.insert(metric_t(3, 211011, 1, 0.608985f, to_vector(rates1))); + metrics.insert(metric_t(3, 211011, 2, 0.298748f, to_vector(rates2))); + metrics.insert(metric_t(3, 211011, 3, 0.287257f, to_vector(rates3))); + } + /** Get the expected binary data + * + * @param buffer binary data string + */ + template + static void create_binary_data(Collection &buffer) + { + const char tmp[] = + {6 + ,20,2,0,8,0,67,67,67,67,67,67,67,67,65,65,65,65,65,65,65 + ,65,3,0,67,56,3,0,1,0,113,-26,27,63,127,-11,-104,62,36,-71,-4 + ,61,3,0,67,56,3,0,2,0,127,-11,-104,62,89,19,-109,62,-70,107,-55 + ,62,3,0,67,56,3,0,3,0,89,19,-109,62,113,-26,27,63,-45,-34,0 + ,63 + }; + buffer.assign(tmp, tmp+util::length_of(tmp)); + } + }; }}} diff --git a/src/tests/interop/metrics/inc/metric_format_fixtures.h b/src/tests/interop/metrics/inc/metric_format_fixtures.h index 1fb6c54dd..020a489eb 100644 --- a/src/tests/interop/metrics/inc/metric_format_fixtures.h +++ b/src/tests/interop/metrics/inc/metric_format_fixtures.h @@ -33,6 +33,7 @@ namespace illumina{ namespace interop { namespace unittest error_metric_v3, error_metric_v4, error_metric_v5, + error_metric_v6, extended_tile_metric_v1, extended_tile_metric_v2, extended_tile_metric_v3, diff --git a/src/tests/interop/metrics/metric_streams_test.cpp b/src/tests/interop/metrics/metric_streams_test.cpp index 23a69bb44..7fe475e8a 100644 --- a/src/tests/interop/metrics/metric_streams_test.cpp +++ b/src/tests/interop/metrics/metric_streams_test.cpp @@ -52,7 +52,7 @@ void print_actual(std::ostream& out, const std::string& data) { for(size_t i=0;i(metric_t::TYPE) == constants::Tile && TypeParam::VERSION == 2) - return; // This contrived exampled is not supported, it includes things like control metrics, which are ignored - const size_t expected_size = io::compute_buffer_size(metrics); - EXPECT_EQ(tmp.size(), expected_size); + TypeParam::create_expected(metrics); + io::write_interop_to_string(tmp, metrics); + const size_t computed_size = io::compute_buffer_size(metrics); + EXPECT_EQ(tmp.size(), computed_size); } /** Confirm the header size matches what is read