Merge pull request #109 from Illumina/update-changes

Support for new features and bug fixes

CMakeLists.txt

@@ -5,6 +5,12 @@ enable_testing()
 include(${PROJECT_SOURCE_DIR}/cmake/Modules/UseGitVersion.cmake)
 set(ARCHIVE_VERSION "v1.0.16-src")
 
+find_package(OpenMP)
+if(OPENMP_FOUND)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+endif()
+
 
 if (NOT CMAKE_BUILD_TYPE)
     message(STATUS "No build type selected, default to Release")
@@ -70,7 +76,12 @@ if(INTEROP_VERSION)
     set(CPACK_PACKAGE_VERSION_PATCH "${VERSION_PATCH}")
     set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}${PACKAGE_SUFFIX}")
     set(CPACK_PACKAGE_INSTALL_DIRECTORY "interop_${VERSION_SHORT}")
-    set(CPACK_OUTPUT_FILE_PREFIX "interop/${VERSION}")
+    if(NOT PACKAGE_OUTPUT_FILE_PREFIX)
+        set(PACKAGE_OUTPUT_FILE_PREFIX ".")
+    else()
+        get_filename_component(PACKAGE_OUTPUT_FILE_PREFIX ${PACKAGE_OUTPUT_FILE_PREFIX} ABSOLUTE)
+    endif()
+    set(CPACK_OUTPUT_FILE_PREFIX "${PACKAGE_OUTPUT_FILE_PREFIX}/interop/${VERSION}")
 endif()
 
 #Adds the target "package"

cmake/package.nuspec.in

@@ -26,7 +26,7 @@
         <tags>native, Illumina, InterOp, C++, C#, @PLATFORM@, @CSHARP_TYPE@ </tags>
     </metadata>
     <files>
-       <file src="@CONFIG@/csharp_interop.dll" target="lib/net40" />
+       <file src="@CONFIG@/csharp_interop.dll" target="lib\net40" />
        <file src="illumina_interop_@CSHARP_TYPE@_@[email protected]" target="build" />
        <file src="@CONFIG@@SHARED_LIB_SEARCH@" target="build" />
     </files>

docs/src/apps.md

@@ -16,5 +16,6 @@ The InterOp package includes several command line tools to extraction informatio
 | @subpage index_summary "index-summary"  | Generate the SAV Indexing Tab summary table as a CSV text file             |
 | @subpage cyclesim "cyclesim"            | Simulate the InterOps of a run folder at a specific cycle                  |
 | @subpage dumpbin "dumpbin"              | Developer app to help create unit tests by dumping the binary format       |
+| @subpage aggregate "aggregate"          | Aggregate by cycle InterOps                                                |
 
 Note: interop2csv has been deprecated in favor of dumptext

docs/src/calculated_metrics.md

@@ -11,3 +11,10 @@ This section describes each metric shown in the SAV summary tab.
  - @subpage q_metrics_requirement_projected_yield "Projected Yield (G)"
  - @subpage error_metrics_requirement "% Error"
 
+### A note on usable cycles. 
+
+Metrics that average over all cycles in the summary tab, actually average over all "usable cycles".  We define a usable 
+cycle as one that is fully corrected in terms of phasing/prephasing. For this reason, we don’t consider the last cycle 
+of a run as usable because it is not fully corrected. So, we don’t count this cycle toward yield or q30 or error rate. 
+Many aligners drop the last cycle for this reason.
+

docs/src/changes.md

@@ -1,9 +1,26 @@
 # Changes                                               {#changes}
 
-## Master
+## v1.0.18 (Master)
 
 Date       | Description
 ---------- | -----------
+
+## v1.0.17
+
+Date       | Description
+---------- | -----------
+2017-03-13 | Update documents to clarify calculations
+2017-03-02 | IPA-6235: Add is_pair_end to run_info
+2017-03-02 | IPA-6233: Release new non-polymorphic template function interface 
+2017-02-24 | IPA-6189: Fixed compute_buffer_size to handle mutli record format tile
+2017-02-20 | IPA-6178: Fix another bug in flowcell loading from collapsed-q
+2017-02-20 | IPA-6057: Simplify polymorphic functions
+2017-02-18 | IPA-6167: Fix bug in by cycle and flowcell plots for BaseSpace Collapsed Q Metrics
+2017-02-16 | IPA-6059: Fix windows regression tests
+2017-02-16 | IPA-6050: Add parse enum to SWIG binding
+2017-02-16 | IPA-6165: Add summary example
+2017-02-16 | IPA-6027: Support RunInfo writing
+2017-02-01 | IPA-6066: Threaded performance test for reading
 2017-01-24 | Update documentation, fixes for compressed q-metrics
 
 ## v1.0.16

docs/src/example_summary.md

@@ -0,0 +1,31 @@
+Summary {#example_summary}
+==========================
+
+This section introduces the way SAV calculates the summary tab. This introduces a limited set of functions. The key 
+classes used below are:
+
+ - `run_metrics`: model holding the binary InterOp data
+ - `run_summary`: model holding the derived summary metrics
+ - `valid_to_load`: byte array (`std::vector<unsigned char>` or `uchar_vector`) indicating which InterOp files to load
+
+The primary functions used below are:
+
+ - `run_metrics::read`: read the InterOp files from disk
+ - `summarize_run_metrics`: summarize the SAV run metrics
+
+C#
+---
+
+@snippet src/examples/csharp/SummaryExample.cs Reporting Summary Metrics in CSharp
+
+
+C++
+---
+
+The following shows how to calculate the summary metrics:
+
+@snippet src/apps/summary.cpp Reporting Summary Metrics in C++
+
+The following shows the implementation of `read_run_metrics` above and how to read the InterOp data from disk:
+
+@snippet src/apps/inc/application.h Reading a subset of run metrics in C++

docs/src/index.md

@@ -54,7 +54,7 @@ List of InterOp Metric Files
 | [IndexMetricsOut.bin]            | Per tile per lane index sequence information                                                                                                                  |
 | [QMetrics2030Out.bin]            | Per tile per cycle Q20/Q30 scores                                                                                                                             |
 | [QMetricsByLaneOut.bin]          | Per tile per cycle Q-score histogram per lane                                                                                                                 |
-| [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle 
+| [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle                                                                                                                            |
 
 [CorrectedIntMetricsOut.bin]: @ref corrected_intensity "CorrectedIntMetricsOut.bin"
 [ErrorMetricsOut.bin]: @ref error_metric "ErrorMetricsOut.bin"

docs/src/j_example.md

@@ -18,4 +18,5 @@ of the examples in that folder with additional commentary.
  - @subpage tile_report "Reporting Tile Metrics"
  - @subpage extraction_report "Reporting Extraction Metrics"
  - @subpage simple_stats "Reporting simple statistics"
+ - @subpage example_summary "Calculating ab SAV-like Summary"
 

interop/external/rapidxml_print.hpp

@@ -99,6 +99,33 @@ namespace rapidxml
             return false;
         }
 
+        template<class OutIt, class Ch>
+        inline OutIt print_children(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_attributes(OutIt out, const xml_node<Ch> *node, int flags);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_data_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_cdata_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_element_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_declaration_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_comment_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_doctype_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
+        template<class OutIt, class Ch>
+        inline OutIt print_pi_node(OutIt out, const xml_node<Ch> *node, int flags, int indent);
+
         ///////////////////////////////////////////////////////////////////////////
         // Internal printing operations
 
@@ -175,7 +202,7 @@ namespace rapidxml
 
         // Print attributes of the node
         template<class OutIt, class Ch>
-        inline OutIt print_attributes(OutIt out, const xml_node<Ch> *node, int flags)
+        inline OutIt print_attributes(OutIt out, const xml_node<Ch> *node, int)
         {
             for (xml_attribute<Ch> *attribute = node->first_attribute(); attribute; attribute = attribute->next_attribute())
             {

interop/io/format/abstract_metric_format.h

@@ -48,6 +48,12 @@ namespace illumina { namespace interop { namespace io
          * @return size of record in bytes
          */
         virtual size_t record_size(const header_t &header) const = 0;
+        /** Calculate the size of a record
+         *
+         * @param metric_set source set of metrics
+         * @return size of buffer in bytes
+         */
+        virtual size_t buffer_size(const model::metric_base::metric_set<Metric>& metric_set) const=0;
         /** Read all the metrics into a metric set
          *
          * @param in input stream

interop/io/format/metric_format.h

@@ -171,6 +171,15 @@ namespace illumina { namespace interop { namespace io
                                            Layout::VERSION);
             return layout_size;
         }
+        /** Calculate the size of a record
+         *
+         * @param metric_set source set of metrics
+         * @return size of buffer in bytes
+         */
+        size_t buffer_size(const model::metric_base::metric_set<Metric>& metric_set) const
+        {
+            return buffer_size(metric_set, int_constant_type<Layout::MULTI_RECORD>::null());
+        }
 
         /** Calculate the size of a record
          *
@@ -209,6 +218,18 @@ namespace illumina { namespace interop { namespace io
             return Layout::MULTI_RECORD > 0;
         }
 
+    private:
+        typedef typename int_constant_type<0>::pointer_t is_single_record_t;
+        typedef typename int_constant_type<1>::pointer_t is_multi_record_t;
+        size_t buffer_size(const model::metric_base::metric_set<Metric>& metric_set, is_single_record_t)const
+        {
+            return header_size(metric_set) + record_size(metric_set)*metric_set.size();
+        }
+        size_t buffer_size(const model::metric_base::metric_set<Metric>& metric_set, is_multi_record_t)const
+        {
+            return Layout::compute_buffer_size(metric_set);
+        }
+
     private:
         static bool test_stream(std::istream& in,
                          const offset_map_t& metric_offset_map,

interop/io/metric_file_stream.h

@@ -30,11 +30,7 @@ namespace illumina { namespace interop { namespace io
     template<class MetricSet>
     size_t compute_buffer_size(const MetricSet& metrics) throw(invalid_argument, bad_format_exception)
     {
-        typedef typename MetricSet::metric_type metric_t;
-        if(is_multi_record(metrics))
-            throw invalid_argument("Format does not currently support computing the buffer size");
-        return header_size(metrics, metrics.version()) +
-                size_of_record<metric_t>(metrics, metrics.version()) * metrics.size();
+        return size_of_buffer(metrics);
     }
     /** Write the metric to a binary byte buffer
      *

interop/io/metric_stream.h

@@ -191,6 +191,30 @@ namespace illumina { namespace interop { namespace io
     }
 
 
+    /** Get the size of a metric file header
+     *
+     * @param metric_set set of metrics
+     * @param version version of the format
+     * @return size of metric file header
+     */
+    template<class MetricSet>
+    size_t size_of_buffer(const MetricSet &metric_set,
+                          ::int16_t version=-1)
+    {
+        typedef typename MetricSet::metric_type metric_t;
+        typedef metric_format_factory<metric_t> factory_type;
+        typedef typename factory_type::metric_format_map metric_format_map;
+        if(version < 1) version = metric_set.version();
+        metric_format_map &format_map = factory_type::metric_formats();
+        if (format_map.find(version) == format_map.end())
+            INTEROP_THROW(bad_format_exception, "No format found to write file with version: "
+                    << version <<  " of " << format_map.size());
+
+        INTEROP_ASSERT(format_map[version]);
+        return format_map[version]->buffer_size(metric_set);
+    }
+
+
     /** Get the size of a metric file header
      *
      * @param header header for metric

interop/io/plot/gnuplot.h

@@ -46,6 +46,7 @@ namespace illumina { namespace interop { namespace io { namespace plot
             out << "set autoscale fix\n";
             out << "set size ratio 2\n";
             out << "set yrange[:] reverse\n";
+            out << "set datafile separator \",\"\n";
             out << "plot \"-\" matrix with image" << "\n";
             const size_t swath_count = data.column_count() / data.tile_count();
             for (size_t y = 0; y < data.tile_count(); ++y) // Rows
@@ -54,15 +55,18 @@ namespace illumina { namespace interop { namespace io { namespace plot
                 {
                     for (size_t s = 0; s < swath_count; ++s) // Columns
                     {
-                        out << " " << table::handle_nan(data(x, y + s * data.tile_count()));
+                        if(x > 0 || s > 0) out << ",";
+                        out << table::handle_nan(data(x, y + s * data.tile_count()));
                     }
-                    out << " nan";
+                    out << ",nan";
                 }
                 out << "\n";
             }
         }
 
         /** Write the flowcell heat map to the output stream using the GNUPlot format
+         *
+         * @note, not currently supported!
          *
          * @param out output stream
          * @param data flowcell heatmap data
@@ -107,13 +111,14 @@ namespace illumina { namespace interop { namespace io { namespace plot
             write_axes(out, data.xyaxes());
             out << "set view map\n";
             out << "set palette defined (0 \"white\", 0.333 \"green\", 0.667 \"yellow\", 1 \"red\")\n";
+            out << "set datafile separator \",\"\n";
             out << "plot \"-\" matrix with image" << "\n";
             for (size_t y = 0; y < data.column_count(); ++y)
             {
                 out << data(0, y);
                 for (size_t x = 1; x < data.row_count(); ++x)
                 {
-                    out << " " << table::handle_nan(data(x, y));
+                    out << "," << table::handle_nan(data(x, y));
                 }
                 out << "\n";
             }
@@ -164,6 +169,7 @@ namespace illumina { namespace interop { namespace io { namespace plot
         {
             if (data.size() == 0) return;
 
+            out << "set datafile separator \",\"\n";
             switch (data[0].series_type())
             {
                 case model::plot::series<Point>::Bar:
@@ -252,11 +258,11 @@ namespace illumina { namespace interop { namespace io { namespace plot
             {
                 for (size_t i = 0; i < series.size(); ++i)
                 {
-                    out << table::handle_nan(series[i].x()) << "\t";
-                    out << table::handle_nan(series[i].lower()) << "\t";
-                    out << table::handle_nan(series[i].p25()) << "\t";
-                    out << table::handle_nan(series[i].p50()) << "\t";
-                    out << table::handle_nan(series[i].p75()) << "\t";
+                    out << table::handle_nan(series[i].x()) << ",";
+                    out << table::handle_nan(series[i].lower()) << ",";
+                    out << table::handle_nan(series[i].p25()) << ",";
+                    out << table::handle_nan(series[i].p50()) << ",";
+                    out << table::handle_nan(series[i].p75()) << ",";
                     out << table::handle_nan(series[i].upper());
                     out << std::endl;
                 }
@@ -274,7 +280,7 @@ namespace illumina { namespace interop { namespace io { namespace plot
         {
             for (size_t i = 0; i < series.size(); ++i)
             {
-                out << table::handle_nan(series[i].x()) << "\t" << table::handle_nan(series[i].y()) << std::endl;
+                out << table::handle_nan(series[i].x()) << "," << table::handle_nan(series[i].y()) << std::endl;
             }
         }
 
@@ -298,8 +304,8 @@ namespace illumina { namespace interop { namespace io { namespace plot
         {
             for (size_t i = 0; i < series.size(); ++i)
             {
-                out << table::handle_nan(series[i].x()) << "\t"
-                    << table::handle_nan(series[i].y()) << "\t"
+                out << table::handle_nan(series[i].x()) << ","
+                    << table::handle_nan(series[i].y()) << ","
                     << table::handle_nan(series[i].width()) << std::endl;
             }
         }