Merge pull request #102 from Illumina/doc-update

Update to documentation and a few minor bug fixes
Illumina · Jan 25, 2017 · afbeaf8 · afbeaf8
2 parents 303bf19 + 33e6eca
commit afbeaf8
Show file tree

Hide file tree

Showing 11 changed files with 289 additions and 30 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,7 +3,7 @@ project(InterOp)
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
 enable_testing()
 include(${PROJECT_SOURCE_DIR}/cmake/Modules/UseGitVersion.cmake)
-set(ARCHIVE_VERSION "v1.0.13-src")
+set(ARCHIVE_VERSION "v1.0.16-src")
 
 
 if (NOT CMAKE_BUILD_TYPE)

diff --git a/README.md b/README.md
@@ -9,6 +9,11 @@ The Illumina InterOp libraries are a set of common routines used for reading Int
 Illumina sequencers. These libraries are backwards compatible and capable of supporting prior releases of the software,
 with one exception: GA systems have been excluded.
 
+***
+>  We are happy to announce support for Illumina's newest sequencer: NovaSeq. This includes support for new InterOp
+>  formats, per cycle InterOp files, and per cycle phasing weights (EmpiricalPhasingMetricsOut.bin).
+***
+
 The InterOp files supported by this library include:
   - CorrectedIntMetricsOut.bin
   - ErrorMetricsOut.bin
@@ -19,6 +24,7 @@ The InterOp files supported by this library include:
   - IndexMetricsOut.bin
   - QMetrics2030Out.bin
   - QMetricsByLaneOut.bin
+  - EmpiricalPhasingMetricsOut.bin
 
 This library is written in C++98 and provides bindings for C# using the SWIG interface. This
 support includes examples and unit tests for both languages.

diff --git a/docs/src/changes.md b/docs/src/changes.md
@@ -1,5 +1,11 @@
 # Changes                                               {#changes}
 
+## Master
+
+Date       | Description
+---------- | -----------
+2017-01-24 | Update documentation, fixes for compressed q-metrics
+
 ## v1.0.16
 
 Date       | Description

diff --git a/docs/src/index.md b/docs/src/index.md
@@ -9,6 +9,11 @@ metric files are binary files produced during a run providing detailed statistic
 metric files are produced after a run during secondary analysis (index metrics) or for faster display of a subset of 
 the original data (collapsed quality scores).
 
+***
+>  We are happy to announce support for Illumina's newest sequencer: NovaSeq. This includes support for new InterOp
+>  formats, per cycle InterOp files, and per cycle phasing weights (EmpiricalPhasingMetricsOut.bin).
+***
+
 Compatibility
 -------------
 
@@ -38,17 +43,18 @@ Table of Contents
 List of InterOp Metric Files
 ----------------------------
 
-| Filename                      | Description                                                                                                                                                   |
-| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [CorrectedIntMetricsOut.bin]  | Per tile per cycle per channel average intensity values                                                                                                       |
-| [ErrorMetricsOut.bin]         | Per tile per cycle (after alignment cycle, e.g. 25) percentages of incorrect calls for aligned clusters                                                       |
-| [ExtractionMetricsOut.bin]    | Per tile per cycle per channel 90th percentile of intensity and FWHM values                                                                                   |
-| [ImageMetricsOut.bin]         | Per tile per cycle per channel percentiles of pixel values used to autocontrast thumbnail images                                                              |
-| [QMetricsOut.bin]             | Per tile per cycle subsampled basecall quality score histogram, also contains QScore Binning information in header as of version 5                            |
-| [TileMetricsOut.bin]          | Per tile information about raw density, cluster count, cluster density passed filter, cluster count passed filter, percent aligned, phasing and prephasing    |
-| [IndexMetricsOut.bin]         | Per tile per lane index sequence information                                                                                                                  |
-| [QMetrics2030Out.bin]         | Per tile per cycle Q20/Q30 scores                                                                                                                             |
-| [QMetricsByLaneOut.bin]       | Per tile per cycle Q-score histogram per lane                                                                                                                 |
+| Filename                         | Description                                                                                                                                                   |
+| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [CorrectedIntMetricsOut.bin]     | Per tile per cycle per channel average intensity values                                                                                                       |
+| [ErrorMetricsOut.bin]            | Per tile per cycle (after alignment cycle, e.g. 25) percentages of incorrect calls for aligned clusters                                                       |
+| [ExtractionMetricsOut.bin]       | Per tile per cycle per channel 90th percentile of intensity and FWHM values                                                                                   |
+| [ImageMetricsOut.bin]            | Per tile per cycle per channel percentiles of pixel values used to autocontrast thumbnail images                                                              |
+| [QMetricsOut.bin]                | Per tile per cycle subsampled basecall quality score histogram, also contains QScore Binning information in header as of version 5                            |
+| [TileMetricsOut.bin]             | Per tile information about raw density, cluster count, cluster density passed filter, cluster count passed filter, percent aligned, phasing and prephasing    |
+| [IndexMetricsOut.bin]            | Per tile per lane index sequence information                                                                                                                  |
+| [QMetrics2030Out.bin]            | Per tile per cycle Q20/Q30 scores                                                                                                                             |
+| [QMetricsByLaneOut.bin]          | Per tile per cycle Q-score histogram per lane                                                                                                                 |
+| [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle 
 
 [CorrectedIntMetricsOut.bin]: @ref corrected_intensity "CorrectedIntMetricsOut.bin"
 [ErrorMetricsOut.bin]: @ref error_metric "ErrorMetricsOut.bin"
@@ -59,6 +65,7 @@ List of InterOp Metric Files
 [IndexMetricsOut.bin]: @ref index_metric "IndexMetricsOut.bin"
 [QMetrics2030Out.bin]: @ref q_collapsed_metric "QMetrics2030Out.bin"
 [QMetricsByLaneOut.bin]: @ref q_metric_by_lane "QMetricsByLaneOut.bin"
+[EmpiricalPhasingMetricsOut.bin]: @ref phasing_metric "EmpiricalPhasingMetricOut.bin"
 
 
 Known Limitations

diff --git a/src/apps/summary.cpp b/src/apps/summary.cpp
@@ -54,6 +54,7 @@ using namespace illumina::interop;
  *
  * @param out output stream
  * @param summary summary metrics
+ * @param information_level level of information to print
  */
 void print_summary(std::ostream& out, const run_summary& summary, const size_t information_level);
 

diff --git a/src/interop/model/metrics/q_collapsed_metric.cpp b/src/interop/model/metrics/q_collapsed_metric.cpp
@@ -104,10 +104,10 @@ namespace illumina{ namespace interop{ namespace io {
             }
             else
             {
-                if(count != TOTAL_RECORD_SIZE )
-                    INTEROP_THROW( incomplete_file_exception, "Insufficient data read from the file, got: "<<
-                            count << " != expected: " << TOTAL_RECORD_SIZE );
-                return ALT_RECORD_SIZE;
+                if(count != RECORD_SIZE )
+                    INTEROP_THROW( incomplete_file_exception, "Insufficient data read from the file, got: "
+                            << count << " != expected: " << RECORD_SIZE );
+                return RECORD_SIZE;
             }
             return count;
         }
@@ -116,11 +116,12 @@ namespace illumina{ namespace interop{ namespace io {
          *
          * @note The size of the record is not fixed for this format, the median q-score may more may not be there
          *
-         * @return 1
+         * @param header collapsed q metric header
+         * @return record size
          */
-        static record_size_t compute_size(const q_collapsed_metric::header_type&)
+        static record_size_t compute_size(const q_collapsed_metric::header_type& header)
         {
-            return static_cast<record_size_t>(ALT_RECORD_SIZE);
+            return static_cast<record_size_t>(header.m_record_size == 0 ? ALT_RECORD_SIZE : header.m_record_size);
         }
 
         /** Compute header size
@@ -309,7 +310,7 @@ namespace illumina{ namespace interop{ namespace io {
      *      2. Version: 5
      */
     template<>
-    struct generic_layout<q_collapsed_metric, 5> : public default_layout<5, 1 /*Tmp hack */>
+    struct generic_layout<q_collapsed_metric, 5> : public default_layout<5>
     {
         /** @page q_collapsed_v5 Collapsed Q-Metrics Version 5
          *
@@ -399,21 +400,22 @@ namespace illumina{ namespace interop{ namespace io {
             }
             else
             {
-                if (count != TOTAL_RECORD_SIZE)
+                if (count != RECORD_SIZE)
                     INTEROP_THROW(incomplete_file_exception, "Insufficient data read from the file, got: " <<
-                                                             count << " != expected: " << TOTAL_RECORD_SIZE);
-                return ALT_RECORD_SIZE;
+                                                             count << " != expected: " << RECORD_SIZE);
+                return RECORD_SIZE;
             }
             return count;
         }
 
         /** Compute the layout size
          *
+         * @param header collapsed q metric header
          * @return size of the record
          */
-        static record_size_t compute_size(const q_collapsed_metric::header_type &)
+        static record_size_t compute_size(const q_collapsed_metric::header_type & header)
         {
-            return static_cast<record_size_t>(ALT_RECORD_SIZE);
+            return static_cast<record_size_t>(header.m_record_size == 0 ? ALT_RECORD_SIZE : header.m_record_size);
         }
         /** Map reading/writing a header to a stream
          *

diff --git a/src/interop/model/metrics/q_metric.cpp b/src/interop/model/metrics/q_metric.cpp
@@ -82,6 +82,8 @@ namespace illumina { namespace interop { namespace io
         template<class Stream, class Metric, class Header>
         static std::streamsize map_stream(Stream &stream, Metric &metric, Header &, const bool)
         {
+            if(metric.size() < q_metric::MAX_Q_BINS)
+                INTEROP_THROW(bad_format_exception, "Cannot write out binned q-score histogram in an unbinned format");
             const std::streamsize count = stream_map<count_t>(stream, metric.m_qscore_hist, q_metric::MAX_Q_BINS);
             resize_accumulated(stream, metric);
             return count;

diff --git a/src/tests/interop/metrics/inc/metric_format_fixtures.h b/src/tests/interop/metrics/inc/metric_format_fixtures.h
@@ -44,6 +44,8 @@ namespace illumina{ namespace interop { namespace unittest
                     q_collapsed_metric_v4,
                     q_collapsed_metric_v5,
                     q_collapsed_metric_v6,
+                    q_collapsed_metric_no_median_v2,
+                    q_collapsed_metric_no_median_v5,
                     q_metric_v4,
                     q_metric_v5,
                     q_metric_v6,

diff --git a/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h b/src/tests/interop/metrics/inc/q_collapsed_metrics_test.h
@@ -92,9 +92,9 @@ namespace illumina{ namespace interop { namespace unittest
         typedef typename parent_t::metric_t metric_t;
         enum{
             /** Do not check the expected binary data */
-            disable_binary_data=true,
+            disable_binary_data=true, // work around for dropping header
             /** Do not check the expected binary data size */
-            disable_binary_data_size=true
+            disable_binary_data_size=true // work around for dropping header
         };
         /** Create the expected metric set
          *
@@ -151,6 +151,147 @@ namespace illumina{ namespace interop { namespace unittest
      */
     struct q_collapsed_metric_v6 : q_collapsed_metric_v5_6<6>{};
 
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     */
+    template<int Version>
+    struct q_collapsed_metric_no_median_v_2_4 : metric_test<model::metrics::q_collapsed_metric, Version>
+    {
+        enum{
+            /** Do not check the expected binary data */
+            disable_binary_data=true,
+            /** Do not check the expected binary data size */
+            disable_binary_data_size=true
+        };
+        /** Define a parent type */
+        typedef metric_test<model::metrics::q_collapsed_metric, Version> parent_t;
+        /** Define a metric set type */
+        typedef typename parent_t::metric_set_t metric_set_t;
+        /** Define a metric type */
+        typedef typename parent_t::metric_t metric_t;
+        /** Create the expected metric set
+         *
+         * @param metrics destination metric set
+         */
+        static void create_expected(metric_set_t& metrics, const model::run::info& =model::run::info())
+        {
+            metrics = metric_set_t(parent_t::VERSION);
+            metrics.insert(metric_t(1,1105,1,2447414,2334829,2566750,0));
+            metrics.insert(metric_t(1,1103,1,2436317,2327796,2543605,0));
+            metrics.insert(metric_t(1,1106,1,2474217,2366046,2583629,0));
+        }
+        /** Get the expected binary data
+         *
+         * @param buffer binary data string
+         */
+        template<class Collection>
+        static void create_binary_data(Collection& buffer)
+        {
+            const int tmp[] =
+            {
+                Version,18
+                ,1,0,81,4,1,0,54,88,37,0,109,-96,35,0,94,42,39,0
+                ,1,0,79,4,1,0,-35,44,37,0,-12,-124,35,0,-11,-49,38,0
+                ,1,0,82,4,1,0,-23,-64,37,0,94,26,36,0,77,108,39,0
+            };
+            buffer.assign(tmp, tmp+util::length_of(tmp));
+        }
+    };
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     * @note Version 2
+     */
+    struct q_collapsed_metric_no_median_v2 : q_collapsed_metric_no_median_v_2_4<2>{};
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     * @note Version 3
+     */
+    struct q_collapsed_metric_no_median_v3 : q_collapsed_metric_no_median_v_2_4<3>{};
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     * @note Version 4
+     */
+    struct q_collapsed_metric_no_median_v4 : q_collapsed_metric_no_median_v_2_4<4>{};
+
+
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     * @note Version 6
+     */
+    template<int Version>
+    struct q_collapsed_metric_no_median_v5_6 : metric_test<model::metrics::q_collapsed_metric, Version>
+    {
+        /** Define a parent type */
+        typedef metric_test<model::metrics::q_collapsed_metric, Version> parent_t;
+        /** Define a metric set type */
+        typedef typename parent_t::metric_set_t metric_set_t;
+        /** Define a metric type */
+        typedef typename parent_t::metric_t metric_t;
+        enum{
+            /** Do not check the expected binary data */
+            disable_binary_data=true,
+            /** Do not check the expected binary data size */
+            disable_binary_data_size=true
+        };
+        /** Create the expected metric set
+         *
+         * @param metrics destination metric set
+         */
+        static void create_expected(metric_set_t& metrics, const model::run::info& =model::run::info())
+        {
+            typedef typename metric_set_t::header_type header_t;
+            typedef typename header_t::qscore_bin_vector_type qscore_bin_vector_type;
+            typedef typename header_t::bin_t bin_t;
+            typedef typename bin_t::bin_type ushort_t;
+            typedef typename metric_t::uint_t uint_t;
+            const uint_t bin_count = 7;
+
+            const ushort_t lower[] = {2, 10, 20, 25, 30, 35, 40};
+            const ushort_t upper[] = {9, 19, 24, 29, 34, 39, 40};
+            const ushort_t value[] = {2, 14, 21, 27, 32, 36, 40};
+            qscore_bin_vector_type headervec;
+            for(uint_t i=0;i<bin_count;i++)
+                headervec.push_back(bin_t(lower[i], upper[i], value[i]));
+
+            metrics = metric_set_t(header_t(headervec), parent_t::VERSION);
+            metrics.insert(metric_t(1,1105,1,2447414,2334829,2566750,0));
+            metrics.insert(metric_t(1,1103,1,2436317,2327796,2543605,0));
+            metrics.insert(metric_t(1,1106,1,2474217,2366046,2583629,0));
+        }
+        /** Get the expected binary data
+         *
+         * @param buffer binary data string
+         */
+        template<class Collection>
+        static void create_binary_data(Collection& buffer)
+        {
+            const int tmp[] =
+            {
+                Version,18,1,7,2,10,20,25,30,35,40,9,19,24,29,34,39,40,2,14,21,27,32,36,40
+                ,1,0,81,4,1,0,54,88,37,0,109,-96,35,0,94,42,39,0
+                ,1,0,79,4,1,0,-35,44,37,0,-12,-124,35,0,-11,-49,38,0
+                ,1,0,82,4,1,0,-23,-64,37,0,94,26,36,0,77,108,39,0
+            };
+            buffer.assign(tmp, tmp+util::length_of(tmp));
+        }
+    };
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     * @note Version 5
+     */
+    struct q_collapsed_metric_no_median_v5 : q_collapsed_metric_no_median_v5_6<5>{};
+    /** This generator creates an expected metric set and the corresponding binary data
+     *
+     * @see model::metrics::q_collapsed_metric
+     * @note Version 6
+     */
+    struct q_collapsed_metric_no_median_v6 : q_collapsed_metric_no_median_v5_6<6>{};
 
 }}}