diff --git a/CMakeLists.txt b/CMakeLists.txt index 28cbfc8da..ac64ccf2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,12 @@ enable_testing() include(${PROJECT_SOURCE_DIR}/cmake/Modules/UseGitVersion.cmake) set(ARCHIVE_VERSION "v1.0.16-src") +find_package(OpenMP) +if(OPENMP_FOUND) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") +endif() + if (NOT CMAKE_BUILD_TYPE) message(STATUS "No build type selected, default to Release") @@ -70,7 +76,12 @@ if(INTEROP_VERSION) set(CPACK_PACKAGE_VERSION_PATCH "${VERSION_PATCH}") set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}${PACKAGE_SUFFIX}") set(CPACK_PACKAGE_INSTALL_DIRECTORY "interop_${VERSION_SHORT}") - set(CPACK_OUTPUT_FILE_PREFIX "interop/${VERSION}") + if(NOT PACKAGE_OUTPUT_FILE_PREFIX) + set(PACKAGE_OUTPUT_FILE_PREFIX ".") + else() + get_filename_component(PACKAGE_OUTPUT_FILE_PREFIX ${PACKAGE_OUTPUT_FILE_PREFIX} ABSOLUTE) + endif() + set(CPACK_OUTPUT_FILE_PREFIX "${PACKAGE_OUTPUT_FILE_PREFIX}/interop/${VERSION}") endif() #Adds the target "package" diff --git a/cmake/package.nuspec.in b/cmake/package.nuspec.in index d45d37b3a..26fee5758 100644 --- a/cmake/package.nuspec.in +++ b/cmake/package.nuspec.in @@ -26,7 +26,7 @@ native, Illumina, InterOp, C++, C#, @PLATFORM@, @CSHARP_TYPE@ - + diff --git a/docs/src/apps.md b/docs/src/apps.md index e4773224b..03b45ed29 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -16,5 +16,6 @@ The InterOp package includes several command line tools to extraction informatio | @subpage index_summary "index-summary" | Generate the SAV Indexing Tab summary table as a CSV text file | | @subpage cyclesim "cyclesim" | Simulate the InterOps of a run folder at a specific cycle | | @subpage dumpbin "dumpbin" | Developer app to help create unit tests by dumping the binary format | +| @subpage aggregate "aggregate" | Aggregate by cycle InterOps | Note: interop2csv has been deprecated in favor of dumptext diff --git a/docs/src/calculated_metrics.md b/docs/src/calculated_metrics.md index 29c6a495d..587b44c53 100644 --- a/docs/src/calculated_metrics.md +++ b/docs/src/calculated_metrics.md @@ -11,3 +11,10 @@ This section describes each metric shown in the SAV summary tab. - @subpage q_metrics_requirement_projected_yield "Projected Yield (G)" - @subpage error_metrics_requirement "% Error" +### A note on usable cycles. + +Metrics that average over all cycles in the summary tab, actually average over all "usable cycles". We define a usable +cycle as one that is fully corrected in terms of phasing/prephasing. For this reason, we don’t consider the last cycle +of a run as usable because it is not fully corrected. So, we don’t count this cycle toward yield or q30 or error rate. +Many aligners drop the last cycle for this reason. + diff --git a/docs/src/changes.md b/docs/src/changes.md index a8c2639b1..dd844e473 100644 --- a/docs/src/changes.md +++ b/docs/src/changes.md @@ -1,9 +1,26 @@ # Changes {#changes} -## Master +## v1.0.18 (Master) Date | Description ---------- | ----------- + +## v1.0.17 + +Date | Description +---------- | ----------- +2017-03-13 | Update documents to clarify calculations +2017-03-02 | IPA-6235: Add is_pair_end to run_info +2017-03-02 | IPA-6233: Release new non-polymorphic template function interface +2017-02-24 | IPA-6189: Fixed compute_buffer_size to handle mutli record format tile +2017-02-20 | IPA-6178: Fix another bug in flowcell loading from collapsed-q +2017-02-20 | IPA-6057: Simplify polymorphic functions +2017-02-18 | IPA-6167: Fix bug in by cycle and flowcell plots for BaseSpace Collapsed Q Metrics +2017-02-16 | IPA-6059: Fix windows regression tests +2017-02-16 | IPA-6050: Add parse enum to SWIG binding +2017-02-16 | IPA-6165: Add summary example +2017-02-16 | IPA-6027: Support RunInfo writing +2017-02-01 | IPA-6066: Threaded performance test for reading 2017-01-24 | Update documentation, fixes for compressed q-metrics ## v1.0.16 diff --git a/docs/src/example_summary.md b/docs/src/example_summary.md new file mode 100644 index 000000000..ce45ba715 --- /dev/null +++ b/docs/src/example_summary.md @@ -0,0 +1,31 @@ +Summary {#example_summary} +========================== + +This section introduces the way SAV calculates the summary tab. This introduces a limited set of functions. The key +classes used below are: + + - `run_metrics`: model holding the binary InterOp data + - `run_summary`: model holding the derived summary metrics + - `valid_to_load`: byte array (`std::vector` or `uchar_vector`) indicating which InterOp files to load + +The primary functions used below are: + + - `run_metrics::read`: read the InterOp files from disk + - `summarize_run_metrics`: summarize the SAV run metrics + +C# +--- + +@snippet src/examples/csharp/SummaryExample.cs Reporting Summary Metrics in CSharp + + +C++ +--- + +The following shows how to calculate the summary metrics: + +@snippet src/apps/summary.cpp Reporting Summary Metrics in C++ + +The following shows the implementation of `read_run_metrics` above and how to read the InterOp data from disk: + +@snippet src/apps/inc/application.h Reading a subset of run metrics in C++ diff --git a/docs/src/index.md b/docs/src/index.md index db5eadacd..caad9aefc 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -54,7 +54,7 @@ List of InterOp Metric Files | [IndexMetricsOut.bin] | Per tile per lane index sequence information | | [QMetrics2030Out.bin] | Per tile per cycle Q20/Q30 scores | | [QMetricsByLaneOut.bin] | Per tile per cycle Q-score histogram per lane | -| [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle +| [EmpiricalPhasingMetricsOut.bin] | Phasing weights per tile per cycle | [CorrectedIntMetricsOut.bin]: @ref corrected_intensity "CorrectedIntMetricsOut.bin" [ErrorMetricsOut.bin]: @ref error_metric "ErrorMetricsOut.bin" diff --git a/docs/src/j_example.md b/docs/src/j_example.md index 193abd2a8..6b464ad32 100644 --- a/docs/src/j_example.md +++ b/docs/src/j_example.md @@ -18,4 +18,5 @@ of the examples in that folder with additional commentary. - @subpage tile_report "Reporting Tile Metrics" - @subpage extraction_report "Reporting Extraction Metrics" - @subpage simple_stats "Reporting simple statistics" + - @subpage example_summary "Calculating ab SAV-like Summary" diff --git a/interop/external/rapidxml_print.hpp b/interop/external/rapidxml_print.hpp index b16dc3ccf..9a4922151 100644 --- a/interop/external/rapidxml_print.hpp +++ b/interop/external/rapidxml_print.hpp @@ -99,6 +99,33 @@ namespace rapidxml return false; } + template + inline OutIt print_children(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_attributes(OutIt out, const xml_node *node, int flags); + + template + inline OutIt print_data_node(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_cdata_node(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_element_node(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_declaration_node(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_comment_node(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_doctype_node(OutIt out, const xml_node *node, int flags, int indent); + + template + inline OutIt print_pi_node(OutIt out, const xml_node *node, int flags, int indent); + /////////////////////////////////////////////////////////////////////////// // Internal printing operations @@ -175,7 +202,7 @@ namespace rapidxml // Print attributes of the node template - inline OutIt print_attributes(OutIt out, const xml_node *node, int flags) + inline OutIt print_attributes(OutIt out, const xml_node *node, int) { for (xml_attribute *attribute = node->first_attribute(); attribute; attribute = attribute->next_attribute()) { diff --git a/interop/io/format/abstract_metric_format.h b/interop/io/format/abstract_metric_format.h index 21d89683c..7eaa12bcd 100644 --- a/interop/io/format/abstract_metric_format.h +++ b/interop/io/format/abstract_metric_format.h @@ -48,6 +48,12 @@ namespace illumina { namespace interop { namespace io * @return size of record in bytes */ virtual size_t record_size(const header_t &header) const = 0; + /** Calculate the size of a record + * + * @param metric_set source set of metrics + * @return size of buffer in bytes + */ + virtual size_t buffer_size(const model::metric_base::metric_set& metric_set) const=0; /** Read all the metrics into a metric set * * @param in input stream diff --git a/interop/io/format/metric_format.h b/interop/io/format/metric_format.h index ff1414aaf..f694712ee 100644 --- a/interop/io/format/metric_format.h +++ b/interop/io/format/metric_format.h @@ -171,6 +171,15 @@ namespace illumina { namespace interop { namespace io Layout::VERSION); return layout_size; } + /** Calculate the size of a record + * + * @param metric_set source set of metrics + * @return size of buffer in bytes + */ + size_t buffer_size(const model::metric_base::metric_set& metric_set) const + { + return buffer_size(metric_set, int_constant_type::null()); + } /** Calculate the size of a record * @@ -209,6 +218,18 @@ namespace illumina { namespace interop { namespace io return Layout::MULTI_RECORD > 0; } + private: + typedef typename int_constant_type<0>::pointer_t is_single_record_t; + typedef typename int_constant_type<1>::pointer_t is_multi_record_t; + size_t buffer_size(const model::metric_base::metric_set& metric_set, is_single_record_t)const + { + return header_size(metric_set) + record_size(metric_set)*metric_set.size(); + } + size_t buffer_size(const model::metric_base::metric_set& metric_set, is_multi_record_t)const + { + return Layout::compute_buffer_size(metric_set); + } + private: static bool test_stream(std::istream& in, const offset_map_t& metric_offset_map, diff --git a/interop/io/metric_file_stream.h b/interop/io/metric_file_stream.h index ad351741f..7be71b8e0 100644 --- a/interop/io/metric_file_stream.h +++ b/interop/io/metric_file_stream.h @@ -30,11 +30,7 @@ namespace illumina { namespace interop { namespace io template size_t compute_buffer_size(const MetricSet& metrics) throw(invalid_argument, bad_format_exception) { - typedef typename MetricSet::metric_type metric_t; - if(is_multi_record(metrics)) - throw invalid_argument("Format does not currently support computing the buffer size"); - return header_size(metrics, metrics.version()) + - size_of_record(metrics, metrics.version()) * metrics.size(); + return size_of_buffer(metrics); } /** Write the metric to a binary byte buffer * diff --git a/interop/io/metric_stream.h b/interop/io/metric_stream.h index 9730553c5..cdbddba75 100644 --- a/interop/io/metric_stream.h +++ b/interop/io/metric_stream.h @@ -191,6 +191,30 @@ namespace illumina { namespace interop { namespace io } + /** Get the size of a metric file header + * + * @param metric_set set of metrics + * @param version version of the format + * @return size of metric file header + */ + template + size_t size_of_buffer(const MetricSet &metric_set, + ::int16_t version=-1) + { + typedef typename MetricSet::metric_type metric_t; + typedef metric_format_factory factory_type; + typedef typename factory_type::metric_format_map metric_format_map; + if(version < 1) version = metric_set.version(); + metric_format_map &format_map = factory_type::metric_formats(); + if (format_map.find(version) == format_map.end()) + INTEROP_THROW(bad_format_exception, "No format found to write file with version: " + << version << " of " << format_map.size()); + + INTEROP_ASSERT(format_map[version]); + return format_map[version]->buffer_size(metric_set); + } + + /** Get the size of a metric file header * * @param header header for metric diff --git a/interop/io/plot/gnuplot.h b/interop/io/plot/gnuplot.h index 19cb39b5e..8600df436 100644 --- a/interop/io/plot/gnuplot.h +++ b/interop/io/plot/gnuplot.h @@ -46,6 +46,7 @@ namespace illumina { namespace interop { namespace io { namespace plot out << "set autoscale fix\n"; out << "set size ratio 2\n"; out << "set yrange[:] reverse\n"; + out << "set datafile separator \",\"\n"; out << "plot \"-\" matrix with image" << "\n"; const size_t swath_count = data.column_count() / data.tile_count(); for (size_t y = 0; y < data.tile_count(); ++y) // Rows @@ -54,15 +55,18 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t s = 0; s < swath_count; ++s) // Columns { - out << " " << table::handle_nan(data(x, y + s * data.tile_count())); + if(x > 0 || s > 0) out << ","; + out << table::handle_nan(data(x, y + s * data.tile_count())); } - out << " nan"; + out << ",nan"; } out << "\n"; } } /** Write the flowcell heat map to the output stream using the GNUPlot format + * + * @note, not currently supported! * * @param out output stream * @param data flowcell heatmap data @@ -107,13 +111,14 @@ namespace illumina { namespace interop { namespace io { namespace plot write_axes(out, data.xyaxes()); out << "set view map\n"; out << "set palette defined (0 \"white\", 0.333 \"green\", 0.667 \"yellow\", 1 \"red\")\n"; + out << "set datafile separator \",\"\n"; out << "plot \"-\" matrix with image" << "\n"; for (size_t y = 0; y < data.column_count(); ++y) { out << data(0, y); for (size_t x = 1; x < data.row_count(); ++x) { - out << " " << table::handle_nan(data(x, y)); + out << "," << table::handle_nan(data(x, y)); } out << "\n"; } @@ -164,6 +169,7 @@ namespace illumina { namespace interop { namespace io { namespace plot { if (data.size() == 0) return; + out << "set datafile separator \",\"\n"; switch (data[0].series_type()) { case model::plot::series::Bar: @@ -252,11 +258,11 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t i = 0; i < series.size(); ++i) { - out << table::handle_nan(series[i].x()) << "\t"; - out << table::handle_nan(series[i].lower()) << "\t"; - out << table::handle_nan(series[i].p25()) << "\t"; - out << table::handle_nan(series[i].p50()) << "\t"; - out << table::handle_nan(series[i].p75()) << "\t"; + out << table::handle_nan(series[i].x()) << ","; + out << table::handle_nan(series[i].lower()) << ","; + out << table::handle_nan(series[i].p25()) << ","; + out << table::handle_nan(series[i].p50()) << ","; + out << table::handle_nan(series[i].p75()) << ","; out << table::handle_nan(series[i].upper()); out << std::endl; } @@ -274,7 +280,7 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t i = 0; i < series.size(); ++i) { - out << table::handle_nan(series[i].x()) << "\t" << table::handle_nan(series[i].y()) << std::endl; + out << table::handle_nan(series[i].x()) << "," << table::handle_nan(series[i].y()) << std::endl; } } @@ -298,8 +304,8 @@ namespace illumina { namespace interop { namespace io { namespace plot { for (size_t i = 0; i < series.size(); ++i) { - out << table::handle_nan(series[i].x()) << "\t" - << table::handle_nan(series[i].y()) << "\t" + out << table::handle_nan(series[i].x()) << "," + << table::handle_nan(series[i].y()) << "," << table::handle_nan(series[i].width()) << std::endl; } } diff --git a/interop/logic/utils/metrics_to_load.h b/interop/logic/utils/metrics_to_load.h index 8de176552..9e7505e33 100644 --- a/interop/logic/utils/metrics_to_load.h +++ b/interop/logic/utils/metrics_to_load.h @@ -19,41 +19,56 @@ namespace illumina { namespace interop { namespace logic { namespace utils * * @param group specific metric group to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_metrics_to_load(const constants::metric_group group, std::vector& valid_to_load); + void list_metrics_to_load(const constants::metric_group group, + std::vector& valid_to_load, + const constants::instrument_type instrument=constants::NovaSeq); /** List the required on demand metrics * * @param type specific metric type to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_metrics_to_load(const constants::metric_type type, std::vector& valid_to_load); + void list_metrics_to_load(const constants::metric_type type, + std::vector& valid_to_load, + const constants::instrument_type instrument=constants::NovaSeq); /** List the required on demand metrics * * @param groups collection of specific metric groups to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ void list_metrics_to_load(const std::vector& groups, - std::vector& valid_to_load); + std::vector& valid_to_load, + const constants::instrument_type instrument=constants::NovaSeq); /** List the required on demand metrics * * @param types collection of specific metric types to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ void list_metrics_to_load(const std::vector& types, - std::vector& valid_to_load); + std::vector& valid_to_load, + const constants::instrument_type instrument=constants::NovaSeq); /** List the required on demand metrics * * @param metric_name name of metric value to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_metrics_to_load(const std::string& metric_name, std::vector& valid_to_load) + void list_metrics_to_load(const std::string& metric_name, + std::vector& valid_to_load, + const constants::instrument_type instrument=constants::NovaSeq) throw(model::invalid_metric_type); /** List all required metric groups * * @param groups destination group list + * @param instrument instrument type */ - void list_summary_metric_groups(std::vector& groups); + void list_summary_metric_groups(std::vector& groups, + const constants::instrument_type instrument=constants::NovaSeq); /** List all required metric groups * @@ -65,8 +80,10 @@ namespace illumina { namespace interop { namespace logic { namespace utils /** List all required metric groups for the summary tab * * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_summary_metrics_to_load(std::vector& valid_to_load); + void list_summary_metrics_to_load(std::vector& valid_to_load, + const constants::instrument_type instrument=constants::NovaSeq); /** List all required metric groups for the index tab * diff --git a/interop/model/metrics/q_metric.h b/interop/model/metrics/q_metric.h index d3a0b794a..b2e4cb434 100644 --- a/interop/model/metrics/q_metric.h +++ b/interop/model/metrics/q_metric.h @@ -251,15 +251,15 @@ namespace illumina { namespace interop { namespace model { namespace metrics enum { /** Unique type code for metric */ - TYPE = constants::Q, + TYPE = constants::Q, /** Latest version of the InterOp format */ - LATEST_VERSION = 7 + LATEST_VERSION = 7 }; public: enum { /** Maximum number of q-score bins */ - MAX_Q_BINS = q_score_header::MAX_Q_BINS + MAX_Q_BINS = q_score_header::MAX_Q_BINS }; /** Q-score metric header */ typedef q_score_header header_type; @@ -412,87 +412,126 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @snippet src/examples/example_q_metric.cpp Calculating Total >= Q30 * - * @sa q_score_header::bins() + * @sa q_score_header::index_for_q_value + * @deprecated This function is deprecated in future versions * @param qscore percentage of clusters over the given q-score value * @param bins q-score histogram bins * @return total of clusters over the given q-score */ uint_t total_over_qscore(const uint_t qscore, - const qscore_bin_vector_type &bins = qscore_bin_vector_type()) const + const qscore_bin_vector_type &bins) const { uint_t total_count = 0; - if (bins.size() == 0) - { - if(qscore <= m_qscore_hist.size()) - total_count = std::accumulate(m_qscore_hist.begin() + qscore, m_qscore_hist.end(), 0); - } - else + for (size_t i = 0; i < bins.size(); i++) { - for (size_t i = 0; i < bins.size(); i++) - { - if (bins[i].value() >= qscore) - total_count += m_qscore_hist[i]; - } + if (bins[i].value() >= qscore) + total_count += m_qscore_hist[i]; } return total_count; } /** Number of clusters over the given q-score * - * This calculates over the cumlative histogram. This function either requires the bins from the - * header or the index of the q-value for the first parameter. Note that the header is apart of the - * metric set (q_metrics). + * This calculates over the local histogram. This function takes an index corresponding to + * the q-value of interest. This index is provided by the `index_for_q_value` in the metric header. * - * @sa q_score_header::bins() + * @snippet src/examples/example_q_metric.cpp Calculating Total >= Q30 + * + * @sa q_score_header::index_for_q_value * @param qscore percentage of clusters over the given q-score value - * @param bins q-score histogram bins * @return total of clusters over the given q-score */ - ::uint64_t total_over_qscore_cumulative(const uint_t qscore, - const qscore_bin_vector_type &bins = qscore_bin_vector_type()) const + uint_t total_over_qscore(const size_t qscore_index) const + { + uint_t total_count = 0; + if(qscore_index <= m_qscore_hist.size()) + total_count = std::accumulate(m_qscore_hist.begin() + qscore_index, m_qscore_hist.end(), 0); + return total_count; + } + + /** Number of clusters over the given q-score + * + * This calculates over the local histogram. This function takes an index corresponding to + * the q-value of interest. This index is provided by the `index_for_q_value` in the metric header. + * + * @sa q_score_header::index_for_q_value + * @param qscore_index index of the q-score (for unbinned 29 is Q30) + * @return total of clusters over the given q-score + */ + ::uint64_t total_over_qscore_cumulative(const size_t qscore_index) const { INTEROP_ASSERT(m_qscore_hist_cumulative.size() > 0); ::uint64_t total_count = 0; - if (bins.size() == 0) - { - INTEROP_ASSERT(qscore > 0); - if(qscore <= m_qscore_hist_cumulative.size()) - total_count = std::accumulate(m_qscore_hist_cumulative.begin() + qscore, m_qscore_hist_cumulative.end(), - static_cast< ::uint64_t >(0)); - } - else - { - for (size_t i = 0; i < bins.size(); i++) - { - if (bins[i].value() >= qscore) - total_count += m_qscore_hist_cumulative[i]; - } - } + if(qscore_index <= m_qscore_hist_cumulative.size()) + total_count = std::accumulate(m_qscore_hist_cumulative.begin() + qscore_index, + m_qscore_hist_cumulative.end(), + static_cast< ::uint64_t >(0)); return total_count; } /** Percent of clusters over the given q-score * - * This calculates over the local histogram. This function either requires the bins from the header - * or the index of the q-value for the first parameter. Note that the header is apart of the - * metric set (q_metrics). + * This calculates over the local histogram. This function takes an index corresponding to + * the q-value of interest. This index is provided by the `index_for_q_value` in the metric header. * * @snippet src/examples/example_q_metric.cpp Calculating Percent >= Q30 * * @sa q_score_header::bins() - * @param qscore percentage of clusters over the given q-score value - * @param bins q-score histogram bins + * @param qscore_index index of the q-score (for unbinned 29 is Q30) * @return percent of cluster over the given q-score */ - float percent_over_qscore(const uint_t qscore, - const qscore_bin_vector_type &bins) const + float percent_over_qscore(const size_t qscore_index) const { const float total = static_cast(sum_qscore()); if (total == 0.0f) return std::numeric_limits::quiet_NaN(); - const uint_t total_count = total_over_qscore(qscore, bins); + const uint_t total_count = total_over_qscore(qscore_index); return 100 * total_count / total; } + /** Percent of clusters over the given q-score + * + * This calculates over the local histogram. This function takes an index corresponding to + * the q-value of interest. This index is provided by the `index_for_q_value` in the metric header. + * + * @sa q_score_header::bins() + * @param qscore_index index of the q-score (for unbinned 29 is Q30) + * @param bins q-score histogram bins + * @return percent of cluster over the given q-score + */ + float percent_over_qscore_cumulative(const size_t qscore_index) const + { + INTEROP_ASSERT(m_qscore_hist_cumulative.size() > 0); + const ::uint64_t total = sum_qscore_cumulative(); + if (total == 0) return std::numeric_limits::quiet_NaN(); + const ::uint64_t total_count = total_over_qscore_cumulative(qscore_index); + return 100.0f * total_count / total; + } + + /** Number of clusters over the given q-score + * + * This calculates over the cumlative histogram. This function either requires the bins from the + * header or the index of the q-value for the first parameter. Note that the header is apart of the + * metric set (q_metrics). + * + * @sa q_score_header::bins() + * @deprecated This function will be removed in future versions + * @param qscore percentage of clusters over the given q-score value + * @param bins q-score histogram bins + * @return total of clusters over the given q-score + */ + ::uint64_t total_over_qscore_cumulative(const uint_t qscore, + const qscore_bin_vector_type &bins) const + { + INTEROP_ASSERT(m_qscore_hist_cumulative.size() > 0); + ::uint64_t total_count = 0; + for (size_t i = 0; i < bins.size(); i++) + { + if (bins[i].value() >= qscore) + total_count += m_qscore_hist_cumulative[i]; + } + return total_count; + } + /** Percent of clusters over the given q-score * * This calculates over the local histogram. This function either requires the bins from the header @@ -502,15 +541,18 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @snippet src/examples/example_q_metric.cpp Calculating Percent >= Q30 * * @sa q_score_header::bins() - * @param qscore percentage of clusters over the given q-score value + * @deprecated This function will be removed in a future version. + * @param qscore_index index of the q-score (for unbinned 29 is Q30) + * @param bins q-score histogram bins * @return percent of cluster over the given q-score */ - float percent_over_qscore(const uint_t qscore) const + float percent_over_qscore(const uint_t qscore_index, + const qscore_bin_vector_type &bins) const { const float total = static_cast(sum_qscore()); if (total == 0.0f) return std::numeric_limits::quiet_NaN(); - const uint_t total_count = total_over_qscore(qscore); - return 100 * total_count / total; + const uint_t total_count = total_over_qscore(qscore_index, bins); + return 100.0f * total_count / total; } /** Percent of clusters over the given q-score @@ -520,13 +562,13 @@ namespace illumina { namespace interop { namespace model { namespace metrics * metric set (q_metrics). * * @sa q_score_header::bins() + * @deprecated This function will be removed in a future version. * @param qscore percentage of clusters over the given q-score value * @param bins q-score histogram bins * @return percent of cluster over the given q-score */ float percent_over_qscore_cumulative(const uint_t qscore, - const qscore_bin_vector_type &bins = - qscore_bin_vector_type()) const + const qscore_bin_vector_type &bins) const { INTEROP_ASSERT(m_qscore_hist_cumulative.size() > 0); const ::uint64_t total = sum_qscore_cumulative(); diff --git a/interop/model/run/info.h b/interop/model/run/info.h index c51867a11..4322bdd60 100644 --- a/interop/model/run/info.h +++ b/interop/model/run/info.h @@ -48,6 +48,8 @@ namespace illumina { namespace interop { namespace model { namespace run * * @param name name of the run * @param date date of the run + * @param instrument_name name of the instrument + * @param run_number number of the run * @param version xml file format version * @param flowcell layout of the flowcell * @param channels names of the color channels @@ -56,6 +58,8 @@ namespace illumina { namespace interop { namespace model { namespace run */ info(const std::string &name = "", const std::string &date = "", + const std::string instrument_name="", + const size_t run_number=0, const uint_t version = 0, const flowcell_layout &flowcell = flowcell_layout(), const str_vector_t &channels = str_vector_t(), @@ -63,6 +67,8 @@ namespace illumina { namespace interop { namespace model { namespace run const read_vector_t &reads = read_vector_t()) : m_name(name), m_date(date), + m_instrument_name(instrument_name), + m_run_number(run_number), m_version(version), m_flowcell(flowcell), m_channels(channels), @@ -76,15 +82,20 @@ namespace illumina { namespace interop { namespace model { namespace run /** Constructor * * @param flowcell layout of the flowcell - * @param channels string list of channel names * @param reads description of the reads + * @param channels string list of channel names */ info(const flowcell_layout &flowcell, - const str_vector_t &channels=str_vector_t(), - const read_vector_t &reads=read_vector_t()) : - m_version(0), + const read_vector_t &reads=read_vector_t(), + const str_vector_t &channels=str_vector_t()) : + m_name(""), + m_date(""), + m_instrument_name(""), + m_run_number(0), + m_version(3), m_flowcell(flowcell), m_channels(channels), + m_image_dim(image_dimensions()), m_reads(reads), m_total_cycle_count(0) { @@ -143,6 +154,25 @@ namespace illumina { namespace interop { namespace model { namespace run throw(invalid_run_info_exception); public: + /** Get the name of the instrument + * + * @return name of the instrument + */ + const std::string &instrument_name() const + { return m_instrument_name; } + /** Get the id of the flowcell + * + * @return id of the flowcell + */ + const std::string &flowcell_id() const + { return m_flowcell.barcode(); } + /** Get the number of the run + * + * @return number of the run + */ + size_t run_number() const + { return m_run_number; } + /** Get the name of the run * * @return name of the run @@ -202,6 +232,21 @@ namespace illumina { namespace interop { namespace model { namespace run if (b->is_index()) return true; return false; } + /** Check if the run is a paired end read + * + * @return true if there is more than one non-index read + */ + bool is_paired_end()const + { + size_t non_index_read_count = 0; + + for (read_vector_t::const_iterator b = m_reads.begin(), e = m_reads.end(); b != e; ++b) + { + if (b->is_index()) continue; + ++non_index_read_count; + } + return non_index_read_count > 1; + } /** Test if cycle is last cycle of a read * * @param cycle cycle number to test @@ -315,9 +360,23 @@ namespace illumina { namespace interop { namespace model { namespace run xml::missing_xml_element_exception, xml::xml_parse_exception); + /** Read run information from the given XML file + * + * @param filename xml file + */ + void write(const std::string &filename)const throw(xml::xml_file_not_found_exception,xml::bad_xml_format_exception); + + /** String containing xml data + * + * @param out output stream + */ + void write(std::ostream& out)const throw(xml::bad_xml_format_exception); + private: std::string m_name; std::string m_date; + std::string m_instrument_name; + size_t m_run_number; uint_t m_version; flowcell_layout m_flowcell; str_vector_t m_channels; diff --git a/interop/model/run_metrics.h b/interop/model/run_metrics.h index 02d739207..e4a8fc9c6 100644 --- a/interop/model/run_metrics.h +++ b/interop/model/run_metrics.h @@ -110,8 +110,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** Read binary metrics and XML files from the run folder * * @param run_folder run folder path + * @param thread_count number of threads to use for network loading */ - void read(const std::string &run_folder) throw(xml::xml_file_not_found_exception, + void read(const std::string &run_folder, const size_t thread_count=1) throw(xml::xml_file_not_found_exception, xml::bad_xml_format_exception, xml::empty_xml_format_exception, xml::missing_xml_element_exception, @@ -127,9 +128,13 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @param run_folder run folder path * @param valid_to_load list of metrics to load + * @param thread_count number of threads to use for network loading * @param skip_loaded skip metrics that are already loaded */ - void read(const std::string &run_folder, const std::vector& valid_to_load, const bool skip_loaded=false) + void read(const std::string &run_folder, + const std::vector& valid_to_load, + const size_t thread_count=1, + const bool skip_loaded=false) throw(xml::xml_file_not_found_exception, xml::bad_xml_format_exception, xml::empty_xml_format_exception, @@ -168,8 +173,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** Read RunParameters.xml if necessary * * @param run_folder run folder path + * @param force_load force loading of run parameters */ - size_t read_run_parameters(const std::string &run_folder) throw( + size_t read_run_parameters(const std::string &run_folder, const bool force_load=false) throw( io::file_not_found_exception, xml::xml_file_not_found_exception, xml::bad_xml_format_exception, @@ -258,18 +264,14 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** List all filenames for a specific metric * + * @param group metric group type * @param files destination interop file names (first one is legacy, all subsequent are by cycle) * @param run_folder run folder location */ - template - void list_filenames(std::vector& files, const std::string& run_folder) - throw(invalid_run_info_exception) - { - typedef typename metric_base::metric_set_helper::metric_set_t metric_set_t; - const size_t last_cycle = run_info().total_cycles(); - if( last_cycle == 0 ) INTEROP_THROW(invalid_run_info_exception, "RunInfo is empty"); - io::list_interop_filenames< metric_set_t >(files, run_folder, last_cycle); - } + void list_filenames(const constants::metric_group group, + std::vector& files, + const std::string& run_folder) + throw(invalid_run_info_exception); public: /** Set a given metric set @@ -333,8 +335,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @param run_folder run folder path * @param last_cycle last cycle of run + * @param thread_count number of threads to use for network loading */ - void read_metrics(const std::string &run_folder, const size_t last_cycle) throw( + void read_metrics(const std::string &run_folder, const size_t last_cycle, const size_t thread_count) throw( io::file_not_found_exception, io::bad_format_exception, io::incomplete_file_exception); @@ -348,11 +351,13 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param run_folder run folder path * @param last_cycle last cycle of run * @param valid_to_load boolean vector indicating which files to load + * @param thread_count number of threads to use for network loading * @param skip_loaded skip metrics that are already loaded */ void read_metrics(const std::string &run_folder, const size_t last_cycle, const std::vector& valid_to_load, + const size_t thread_count, const bool skip_loaded=false) throw( io::file_not_found_exception, io::bad_format_exception, @@ -366,6 +371,40 @@ namespace illumina { namespace interop { namespace model { namespace metrics io::file_not_found_exception, io::bad_format_exception); + + /** Read a single metric set from a binary buffer + * + * @param group metric set to write + * @param buffer binary buffer + * @param buffer_size size of binary buffer + */ + void read_metrics_from_buffer(const constants::metric_group group, + uint8_t* buffer, + const size_t buffer_size) throw( + io::file_not_found_exception, + io::bad_format_exception, + io::incomplete_file_exception, + model::index_out_of_bounds_exception); + /** Write a single metric set to a binary buffer + * + * @param group metric set to write + * @param buffer binary buffer + * @param buffer_size size of binary buffer + */ + void write_metrics_to_buffer(const constants::metric_group group, + uint8_t* buffer, + const size_t buffer_size)const throw( + io::invalid_argument, + io::bad_format_exception, + io::incomplete_file_exception); + /** Calculate the required size of the buffer for writing + * + * @param group metric set to write + * @return required size of the binary buffer + */ + size_t calculate_buffer_size(const constants::metric_group group)const throw( + io::invalid_argument, io::bad_format_exception); + /** Validate whether the RunInfo.xml matches the InterOp files * * @throws invalid_run_info_exception @@ -393,6 +432,15 @@ namespace illumina { namespace interop { namespace model { namespace metrics { m_metrics.apply(func); } + /** Read binary metrics from the run folder + * + * @param func call back for metric reading + */ + template + void metrics_callback(Func &func)const + { + m_metrics.apply(func); + } /** Check if the metric group is empty * * @param group_name prefix of interop group metric diff --git a/interop/model/summary/metric_summary.h b/interop/model/summary/metric_summary.h index 927295ed3..982af9b33 100644 --- a/interop/model/summary/metric_summary.h +++ b/interop/model/summary/metric_summary.h @@ -37,9 +37,11 @@ namespace illumina { namespace interop { namespace model { namespace summary { * @ref illumina::interop::model::summary::metric_summary "See full class description" * @{ */ - /** Get the error rate of PHIX for the run + /** Get the error rate of PHIX for the run over all "usable cycles". * - * @return error rate of PHIX for run + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. + * + * @return error rate of PHIX for run over all "usable cycles". */ float error_rate()const { @@ -61,26 +63,32 @@ namespace illumina { namespace interop { namespace model { namespace summary { { return m_first_cycle_intensity; } - /** Get the percent of bases greater than Q30 + /** Get the percent of bases greater than or equal to Q30 over all "usable cycles" + * + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. * - * @return percent of bases greater than Q30 + * @return percent of bases greater than or equal to Q30 over all "usable cycles" */ float percent_gt_q30()const { return m_percent_gt_q30; } - /** Get the yield of the run in gigabases + /** Get the yield of the run in gigabases over all "usable cycles" * - * @return yield of the run in gigabases + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. + * + * @return yield of the run in gigabases over all "usable cycles" */ float yield_g()const { return m_yield_g; } - /** Get the projected yield of teh run in gigabases + /** Get the projected yield of the run in gigabases over all "usable cycles" + * + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. * - * @return projected yield of teh run in gigabases + * @return projected yield of the run in gigabases over all "usable cycles" */ float projected_yield_g()const { @@ -111,9 +119,9 @@ namespace illumina { namespace interop { namespace model { namespace summary { { m_percent_aligned = val; } - /** Set the percent of bases greater than Q30 + /** Set the percent of bases greater than or equal to Q30 * - * @param val percent of bases greater than Q30 + * @param val percent of bases greater than or equal to Q30 */ void percent_gt_q30(const float val) { @@ -128,9 +136,9 @@ namespace illumina { namespace interop { namespace model { namespace summary { { m_yield_g = val; } - /** Set the projected yield of teh run in gigabases + /** Set the projected yield of the run in gigabases * - * @param val projected yield of teh run in gigabases + * @param val projected yield of the run in gigabases */ void projected_yield_g(const float val) { diff --git a/interop/model/summary/stat_summary.h b/interop/model/summary/stat_summary.h index 1ce728173..61aac7bb4 100644 --- a/interop/model/summary/stat_summary.h +++ b/interop/model/summary/stat_summary.h @@ -51,27 +51,33 @@ namespace illumina { namespace interop { namespace model { namespace summary * @ref illumina::interop::model::summary::surface_summary "See full class description" * @{ */ - /** Get the percent of bases greater than Q30 + /** Get the percent of bases greater than or equal to Q30 over all usable cycles * - * @return percent of bases greater than Q30 + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. + * + * @return percent of bases greater than or equal to Q30 over all usable cycles */ float percent_gt_q30() const { return m_percent_gt_q30; } - /** Get the yield of the run in gigabases + /** Get the yield of the run in gigabases over all usable cycles + * + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. * - * @return yield of the run in gigabases + * @return yield of the run in gigabases over all usable cycles */ float yield_g() const { return m_yield_g; } - /** Get the projected yield of the run in gigabases + /** Get the projected yield of the run in gigabases over all usable cycles * - * @return projected yield of the run in gigabases + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. + * + * @return projected yield of the run in gigabases over all usable cycles */ float projected_yield_g() const { @@ -169,45 +175,55 @@ namespace illumina { namespace interop { namespace model { namespace summary return m_percent_aligned; } - /** Get statistics summarizing the PhiX error rate of tiles in the lane + /** Get statistics summarizing the PhiX error rate of tiles in the lane over all usable cycles + * + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. * - * @return statistics summarizing the PhiX error rate of tiles in the lane + * @return statistics summarizing the PhiX error rate of tiles in the lane over all usable cycles */ const metric_stat_t &error_rate() const { return m_error_rate; } - /** Get statistics summarizing the PhiX error rate over the first 35 cycles of tiles in the lane + /** Get statistics summarizing the PhiX error rate over the first 35 usable cycles of tiles in the lane * - * @return statistics summarizing the PhiX error rate over the first 35 cycles of tiles in the lane + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. + * + * @return statistics summarizing the PhiX error rate over the first 35 usable cycles of tiles in the lane */ const metric_stat_t &error_rate_35() const { return m_error_rate_35; } - /** Get statistics summarizing the PhiX error rate over the first 50 cycles of tiles in the lane + /** Get statistics summarizing the PhiX error rate over the first 50 usable cycles of tiles in the lane + * + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. * - * @return statistics summarizing the PhiX error rate over the first 50 cycles of tiles in the lane + * @return statistics summarizing the PhiX error rate over the first 50 usable cycles of tiles in the lane */ const metric_stat_t &error_rate_50() const { return m_error_rate_50; } - /** Get statistics summarizing the PhiX error rate over the first 75 cycles of tiles in the lane + /** Get statistics summarizing the PhiX error rate over the first 75 usable cycles of tiles in the lane * - * @return statistics summarizing the PhiX error rate over the first 75 cycles of tiles in the lane + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. + * + * @return statistics summarizing the PhiX error rate over the first 75 usable cycles of tiles in the lane */ const metric_stat_t &error_rate_75() const { return m_error_rate_75; } - /** Get statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane + /** Get statistics summarizing the PhiX error rate over the first 100 usable cycles of tiles in the lane + * + * A "usable cycle" is a cycle that is fully corrected, so the last cycle of a read is excluded. * - * @return statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane + * @return statistics summarizing the PhiX error rate over the first 100 usable cycles of tiles in the lane */ const metric_stat_t &error_rate_100() const { @@ -258,9 +274,9 @@ namespace illumina { namespace interop { namespace model { namespace summary /** @} */ public: - /** Set the percent of bases greater than Q30 + /** Set the percent of bases greater than or equal to Q30 * - * @param val percent of bases greater than Q30 + * @param val percent of bases greater than or equal to Q30 */ void percent_gt_q30(const float val) { @@ -394,9 +410,9 @@ namespace illumina { namespace interop { namespace model { namespace summary { m_error_rate_75 = stat; } - /** Set statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane + /** Set statistics summarizing the PhiX error rate over the first 100 usable cycles of tiles in the lane * - * @param stat statistics summarizing the PhiX error rate over the first 100 cycles of tiles in the lane + * @param stat statistics summarizing the PhiX error rate over the first 100 usable cycles of tiles in the lane */ void error_rate_100(const metric_stat_t& stat) { diff --git a/interop/util/xml_parser.h b/interop/util/xml_parser.h index 18881ab74..9d2600430 100644 --- a/interop/util/xml_parser.h +++ b/interop/util/xml_parser.h @@ -11,6 +11,7 @@ #include #include +#include #include "interop/util/xml_exceptions.h" #include "interop/util/exception.h" @@ -22,6 +23,83 @@ namespace illumina { namespace interop { namespace xml /** XML attribute pointer type */ typedef rapidxml::xml_attribute<> *xml_attr_ptr; + /** XML document wrapper */ + class xml_document + { + public: + /** Constructor */ + xml_document() + { + rapidxml::xml_node<>* decl = m_doc.allocate_node(rapidxml::node_declaration); + m_backing.reserve(50); + decl->append_attribute(m_doc.allocate_attribute("version", "1.0")); + m_doc.append_node(decl); + } + /** Add a node to the document with a given parent + * + * @param parent parent of the node + * @param name name of the node + * @return pointer to new node + */ + xml_node_ptr add_node(xml_node_ptr parent, const char* name) + { + rapidxml::xml_node<>* child = m_doc.allocate_node(rapidxml::node_element, name); + parent->append_node(child); + return child; + } + /** Add a node to the document root + * + * @param name name of the node + * @return pointer to new node + */ + xml_node_ptr add_node(const char* name) + { + return add_node(&m_doc, name); + } + /** Add a node to the document with a given parent + * + * @param parent parent of the node + * @param name name of the node + * @param val value of the node + * @return pointer to new node + */ + template + xml_node_ptr add_node(xml_node_ptr parent, const char* name, const T& val) + { + m_backing.push_back(util::lexical_cast(val)); + rapidxml::xml_node<>* child = m_doc.allocate_node(rapidxml::node_element, name, m_backing.back().c_str()); + parent->append_node(child); + return child; + } + /** Add an attribute to the given parent node + * + * @param parent parent of the node + * @param name name of the node + * @param val value of the node + * @return pointer to new node + */ + template + void add_attribute(xml_node_ptr parent, const char* name, const T& val) + { + m_backing.push_back(util::lexical_cast(val)); + parent->append_attribute(m_doc.allocate_attribute(name, m_backing.back().c_str())); + } + /** Write an XML document to the output stream + * + * @param out output stream + * @param doc XML document + * @return output stream + */ + friend std::ostream& operator<<(std::ostream& out, const xml_document& doc) + { + out << doc.m_doc; + return out; + } + + private: + rapidxml::xml_document<> m_doc; + std::vector m_backing; + }; /** Check if the xml node matches the target, and, if so, set the value * * @param p_node current node @@ -125,7 +203,12 @@ namespace illumina { namespace interop { namespace xml if (p_attr == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find attribute: " << target); if (p_attr->name() == target) { - val = util::lexical_cast(p_attr->value()); + std::string tmp = p_attr->value(); + if(tmp[0] == '\"' && tmp[tmp.length()-1] == '\"') + { + tmp = tmp.substr(1, tmp.length()-1); + } + val = util::lexical_cast(tmp); return true; } return false; @@ -141,7 +224,12 @@ namespace illumina { namespace interop { namespace xml { if (p_attr == 0) INTEROP_THROW(missing_xml_element_exception, "Cannot find attribute"); - val = util::lexical_cast(p_attr->value()); + std::string tmp = p_attr->value(); + if(tmp[0] == '\"' && tmp[tmp.length()-1] == '\"') + { + tmp = tmp.substr(1, tmp.length()-1); + } + val = util::lexical_cast(tmp); } /** Check if the xml node matches the target, and, if so, save the children to the vector diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt index d3c30823c..fe016ecf7 100644 --- a/src/apps/CMakeLists.txt +++ b/src/apps/CMakeLists.txt @@ -30,7 +30,6 @@ endfunction() set(SWIG_VERSION_INFO "") -add_application(interop2csv interop2csv.cpp) add_application(dumptext dumptext.cpp) add_application(cyclesim cyclesim.cpp) add_application(summary summary.cpp) diff --git a/src/apps/aggregate.cpp b/src/apps/aggregate.cpp index 06130786c..be09bd0fa 100644 --- a/src/apps/aggregate.cpp +++ b/src/apps/aggregate.cpp @@ -17,6 +17,7 @@ #include "interop/util/filesystem.h" #include "interop/model/run_metrics.h" #include "interop/version.h" +#include "interop/util/option_parser.h" #include "inc/application.h" using namespace illumina::interop::model::metrics; @@ -28,6 +29,46 @@ void zero_extraction_time(I beg, I end) for(;beg != end;++beg) beg->date_time(0); } +/** Copy of subset of metrics + */ +struct subset_copier +{ + /** Constructor + * + * @param run run metrics + * @param max_tile_number maximum tile number + */ + subset_copier(run_metrics& run, const size_t max_tile_number) : m_run(run), m_max_tile_number(max_tile_number){} + + /** Function operator overload to collect a subset of metrics + * + * @param metrics set of metrics + */ + template + void operator()(const MetricSet& metrics)const + { + typedef typename MetricSet::base_t base_t; + copy(metrics, base_t::null()); + } + +private: + template + void copy(const MetricSet& metrics, const void*)const + { + m_run.get() = MetricSet(metrics, metrics.version()); + + for(size_t i=0;i m_max_tile_number) continue; + m_run.get().insert(metrics[i]); + } + } + +private: + run_metrics& m_run; + size_t m_max_tile_number; +}; + int main(int argc, const char** argv) { if(argc == 0) @@ -37,26 +78,76 @@ int main(int argc, const char** argv) return INVALID_ARGUMENTS; } + const size_t thread_count = 1; + std::cout << "# Version: " << INTEROP_VERSION << std::endl; + size_t max_tile_number=0; + util::option_parser description; + description + (max_tile_number, "max-tile", "Maximum tile number to include"); + if(description.is_help_requested(argc, argv)) + { + std::cout << "Usage: " << io::basename(argv[0]) << " run_folder [--option1=value1] [--option2=value2]" << std::endl; + description.display_help(std::cout); + return SUCCESS; + } + try + { + description.parse(argc, argv); + description.check_for_unknown_options(argc, argv); + } + catch(const util::option_exception& ex) + { + std::cerr << ex.what() << std::endl; + return INVALID_ARGUMENTS; + } + run_metrics run; const std::string input_file=argv[1]; const std::string run_name = io::basename(input_file); std::cout << "# Run Folder: " << run_name << std::endl; - int ret = read_run_metrics(input_file.c_str(), run); + int ret = read_run_metrics(input_file.c_str(), run, thread_count); if(ret != SUCCESS) return ret; io::mkdir("InterOp"); run.sort(); zero_extraction_time(run.get().begin(), run.get().end()); - try{ - run.write_metrics("."); + if(max_tile_number > 0) + { + run_metrics subset; + subset_copier copy_subset(subset, max_tile_number); + try + { + run.metrics_callback(copy_subset); + } + catch(const std::exception& ex) + { + std::cerr << ex.what() << std::endl; + return UNEXPECTED_EXCEPTION; + } + std::cout << subset.get().size() << ", " << run.get().size() << std::endl; + try{ + subset.write_metrics("."); + } + catch(const std::exception& ex) + { + std::cerr << "Error: Unable to write InterOp files" << std::endl; + std::cerr << ex.what() << std::endl; + return UNEXPECTED_EXCEPTION; + } } - catch(const std::exception& ex) + else { - std::cerr << "Error: Unable to write InterOp files" << std::endl; - std::cerr << ex.what() << std::endl; - return UNEXPECTED_EXCEPTION; + try{ + run.write_metrics("."); + } + catch(const std::exception& ex) + { + std::cerr << "Error: Unable to write InterOp files" << std::endl; + std::cerr << ex.what() << std::endl; + return UNEXPECTED_EXCEPTION; + } } return SUCCESS; } diff --git a/src/apps/dumpbin.cpp b/src/apps/dumpbin.cpp index e9f67177b..699922674 100644 --- a/src/apps/dumpbin.cpp +++ b/src/apps/dumpbin.cpp @@ -135,6 +135,7 @@ int main(int argc, char** argv) return INVALID_ARGUMENTS; } const size_t subset_count=3; + const size_t thread_count = 1; metric_writer write_metrics(std::cout); std::cout << "# Version: " << INTEROP_VERSION << std::endl; @@ -143,7 +144,7 @@ int main(int argc, char** argv) { run_metrics run; run_metrics subset; - int ret = read_run_metrics(argv[i], run); + int ret = read_run_metrics(argv[i], run, thread_count); if(ret != SUCCESS) return ret; subset_copier copy_subset(subset, subset_count); try diff --git a/src/apps/dumptext.cpp b/src/apps/dumptext.cpp index 8cc00a9b4..453515abc 100644 --- a/src/apps/dumptext.cpp +++ b/src/apps/dumptext.cpp @@ -185,6 +185,7 @@ int main(int argc, const char** argv) //print_help(std::cout); return INVALID_ARGUMENTS; } + const size_t thread_count = 1; const char eol = '\n'; @@ -213,7 +214,7 @@ int main(int argc, const char** argv) for(int i=1;i 0 ) diff --git a/src/apps/imaging_table.cpp b/src/apps/imaging_table.cpp index 299e024c4..3113c5cf0 100644 --- a/src/apps/imaging_table.cpp +++ b/src/apps/imaging_table.cpp @@ -38,6 +38,7 @@ int main(int argc, char** argv) std::cerr << "No arguments specified!" << std::endl; return INVALID_ARGUMENTS; } + const size_t thread_count = 1; std::cout << "# Version: " << INTEROP_VERSION << std::endl; @@ -48,7 +49,7 @@ int main(int argc, char** argv) { run_metrics run; std::cout << "# Run Folder: " << io::basename(argv[i]) << std::endl; - int ret = read_run_metrics(argv[i], run, valid_to_load); + int ret = read_run_metrics(argv[i], run, valid_to_load, thread_count); if (ret != SUCCESS) return ret; model::table::imaging_table table; diff --git a/src/apps/inc/application.h b/src/apps/inc/application.h index f322181a3..3fc30cfc7 100644 --- a/src/apps/inc/application.h +++ b/src/apps/inc/application.h @@ -37,18 +37,20 @@ enum exit_codes * * @param filename run folder containing RunInfo.xml and InterOps * @param metrics run metrics + * @param thread_count number of threads to use for network loading * @param check_empty if true return an error if the metrics are empty * @return exit code */ inline int read_run_metrics(const char* filename, illumina::interop::model::metrics::run_metrics& metrics, + const size_t thread_count, const bool check_empty=true) { using namespace illumina::interop; using namespace illumina::interop::model; try { - metrics.read(filename); + metrics.read(filename, thread_count); } catch(const xml::xml_file_not_found_exception& ex) { @@ -84,20 +86,23 @@ inline int read_run_metrics(const char* filename, * @param filename run folder containing RunInfo.xml and InterOps * @param metrics run metrics * @param valid_to_load list of metrics that are valid to load + * @param thread_count number of threads to use for network loading * @param check_empty if true return an error if the metrics are empty * @return exit code */ inline int read_run_metrics(const char* filename, illumina::interop::model::metrics::run_metrics& metrics, const std::vector& valid_to_load, + const size_t thread_count, const bool check_empty=true) { +// @ [Reading a subset of run metrics in C++] using namespace illumina::interop; using namespace illumina::interop::model; try { metrics.clear(); - metrics.read(filename, valid_to_load); + metrics.read(filename, valid_to_load, thread_count); } catch(const xml::xml_file_not_found_exception& ex) { @@ -124,5 +129,6 @@ inline int read_run_metrics(const char* filename, std::cerr << "No InterOp files found" << std::endl; return EMPTY_INTEROP; } +// @ [Reading a subset of run metrics in C++] return SUCCESS; } diff --git a/src/apps/index_summary.cpp b/src/apps/index_summary.cpp index e341cec87..232fd960f 100644 --- a/src/apps/index_summary.cpp +++ b/src/apps/index_summary.cpp @@ -71,6 +71,7 @@ int main(int argc, char** argv) //print_help(std::cout); return INVALID_ARGUMENTS; } + const size_t thread_count = 1; std::vector valid_to_load; logic::utils::list_index_metrics_to_load(valid_to_load); // Only load the InterOp files required @@ -78,7 +79,7 @@ int main(int argc, char** argv) for(int i=1;i data; diff --git a/src/apps/summary.cpp b/src/apps/summary.cpp index de36baee2..0df4290b6 100644 --- a/src/apps/summary.cpp +++ b/src/apps/summary.cpp @@ -67,6 +67,7 @@ int main(int argc, const char** argv) //print_help(std::cout); return INVALID_ARGUMENTS; } + const size_t thread_count = 1; size_t information_level=5; util::option_parser description; @@ -89,6 +90,7 @@ int main(int argc, const char** argv) return INVALID_ARGUMENTS; } +// @ [Reporting Summary Metrics in C++] std::vector valid_to_load; logic::utils::list_summary_metrics_to_load(valid_to_load); // Only load the InterOp files required @@ -98,7 +100,7 @@ int main(int argc, const char** argv) run_metrics run; std::cout << io::basename(argv[i]) << std::endl; - int ret = read_run_metrics(argv[i], run, valid_to_load); + int ret = read_run_metrics(argv[i], run, valid_to_load, thread_count); if(ret != SUCCESS) { continue; @@ -123,6 +125,7 @@ int main(int argc, const char** argv) return UNEXPECTED_EXCEPTION; } } +// @ [Reporting Summary Metrics in C++] return SUCCESS; } @@ -252,6 +255,11 @@ void summarize(const surface_summary& summary, std::vector& values, values[i++] = util::format(summary.phasing().mean(), 3, 3) + " / " + util::format(summary.prephasing().mean(), 3, 3); + values[i++] = util::format(summary.phasing_slope().mean(), 3, 3) + "*c " + (summary.phasing_offset().mean() > 0 ? "+ " : "- ") + + util::format(static_cast(std::abs(summary.phasing_offset().mean())), 3, 3); + values[i++] = util::format(summary.prephasing_slope().mean(), 3, 3) + "*c " + (summary.prephasing_offset().mean() > 0 ? "+ " : "- ") + + util::format(static_cast(std::abs(summary.prephasing_offset().mean())), 3, 3); + values[i++] = format(summary.reads(), 0, 2, 1e6); values[i++] = format(summary.reads_pf(), 0, 2, 1e6); values[i++] = format(summary.percent_gt_q30(), 0, 2); @@ -278,6 +286,10 @@ void summarize(const lane_summary& summary, std::vector& values) values[i++] = format(summary.percent_pf(), 0, 2); values[i++] = util::format(summary.phasing().mean(), 3, 3) + " / " + util::format(summary.prephasing().mean(), 3, 3); + values[i++] = util::format(summary.phasing_slope().mean(), 3, 3) + "*c " + (summary.phasing_offset().mean() > 0 ? "+ " : "- ") + + util::format(static_cast(std::abs(summary.phasing_offset().mean())), 3, 3); + values[i++] = util::format(summary.prephasing_slope().mean(), 3, 3) + "*c " + (summary.prephasing_offset().mean() > 0 ? "+ " : "- ") + + util::format(static_cast(std::abs(summary.prephasing_offset().mean())), 3, 3); values[i++] = format(summary.reads(), 0, 2, 1e6); values[i++] = format(summary.reads_pf(), 0, 2, 1e6); values[i++] = format(summary.percent_gt_q30(), 0, 2); @@ -327,6 +339,7 @@ void print_summary(std::ostream& out, const run_summary& summary, const size_t i if( information_level >= 3) { const char *lane_header[] = {"Lane", "Surface", "Tiles", "Density", "Cluster PF", "Phas/Prephas", + "Phasing Equation", "Prephas Equation", "Reads", "Reads PF", "%>=Q30", "Yield", "Cycles Error", "Aligned", "Error", "Error (35)", "Error (75)", "Error (100)", "Intensity C1"}; values.resize(util::length_of(lane_header)); diff --git a/src/examples/csharp/SummaryExample.cs b/src/examples/csharp/SummaryExample.cs index 59d7722da..d38773c15 100644 --- a/src/examples/csharp/SummaryExample.cs +++ b/src/examples/csharp/SummaryExample.cs @@ -1,5 +1,5 @@ -// @ [Create Summary Model from Run Folder] +// @ [Reporting Summary Metrics in CSharp] using System; using Illumina.InterOp.Run; using Illumina.InterOp.Metrics; @@ -32,4 +32,4 @@ static int Main(string[] args) return 0; } } -// @ [Reading a binary InterOp file in CSharp] +// @ [Reporting Summary Metrics in CSharp] diff --git a/src/examples/example_q_metric.cpp b/src/examples/example_q_metric.cpp index cdf995ea5..2a25458e4 100644 --- a/src/examples/example_q_metric.cpp +++ b/src/examples/example_q_metric.cpp @@ -57,14 +57,14 @@ int main(int argc, char** argv) // @ [Calculating Total >= Q30] q_metric &metric0 = q_metric_set[0]; - std::cout << "Total >= Q30: " << metric0.total_over_qscore(30, q_metric_set.bins()) << std::endl; + std::cout << "Total >= Q30: " << metric0.total_over_qscore(q_metric_set.index_for_q_value(30)) << std::endl; // @ [Calculating Total >= Q30] // @ [Calculating Percent >= Q30] q_metric &metric1 = q_metric_set[0]; - std::cout << "Percent >= Q30: " << metric1.percent_over_qscore(30, q_metric_set.bins()) << std::endl; + std::cout << "Percent >= Q30: " << metric1.percent_over_qscore(q_metric_set.index_for_q_value(30)) << std::endl; // @ [Calculating Percent >= Q30] } diff --git a/src/ext/swig/run.i b/src/ext/swig/run.i index 5f6052de4..8acff3ef1 100644 --- a/src/ext/swig/run.i +++ b/src/ext/swig/run.i @@ -65,6 +65,12 @@ %define WRAP_ENUM(ENUM) %template(list_##ENUM) illumina::interop::constants::list_enum_names< illumina::interop::constants:: ENUM >; +%template(parse_##ENUM) illumina::interop::constants::parse< illumina::interop::constants:: ENUM >; +#if defined(SWIGPYTHON) +%template(to_string_##ENUM) illumina::interop::constants::to_string< illumina::interop::constants:: ENUM >; +#else +%template(to_string) illumina::interop::constants::to_string< illumina::interop::constants:: ENUM >; +#endif %enddef WRAP_ENUM(metric_type) diff --git a/src/ext/swig/run_metrics.i b/src/ext/swig/run_metrics.i index cc6d71329..1a83a0053 100644 --- a/src/ext/swig/run_metrics.i +++ b/src/ext/swig/run_metrics.i @@ -41,6 +41,17 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Metric Logic //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// The SWIG Python binding does not support polymorphic functions +#if defined(SWIGPYTHON) + %rename(list_metrics_to_load_metric_group) illumina::interop::logic::utils::list_metrics_to_load(const constants::metric_group group, + std::vector& valid_to_load, + const constants::instrument_type instrument); + %rename(list_metrics_to_load_metric_groups) illumina::interop::logic::utils::list_metrics_to_load(const std::vector& groups, std::vector& valid_to_load, + const constants::instrument_type type); +#endif + + %{ #include "interop/logic/metric/extraction_metric.h" #include "interop/logic/metric/q_metric.h" @@ -64,7 +75,6 @@ EXCEPTION_WRAPPER(WRAP_EXCEPTION_IMPORT) %include "interop/model/run_metrics.h" %define WRAP_RUN_METRICS(metric_t) - %template(list_ ## metric_t ## _filenames) illumina::interop::model::metrics::run_metrics::list_filenames< metric_t >; %template(set_ ## metric_t ## _set) illumina::interop::model::metrics::run_metrics::set< illumina::interop::model::metric_base::metric_set< metric_t> >; %template(metric_t ## _set) illumina::interop::model::metrics::run_metrics::get_metric_set< metric_t >; %enddef diff --git a/src/interop/logic/plot/plot_by_cycle.cpp b/src/interop/logic/plot/plot_by_cycle.cpp index ed8efd66c..61980b641 100644 --- a/src/interop/logic/plot/plot_by_cycle.cpp +++ b/src/interop/logic/plot/plot_by_cycle.cpp @@ -155,6 +155,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot INTEROP_THROW(model::invalid_filter_option, "Only cycle metrics are supported"); options.validate(type, metrics.run_info()); // TODO: Check ignored? size_t max_cycle=0; + bool is_empty = true; switch(logic::utils::to_group(type)) { case constants::Extraction: @@ -186,6 +187,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot type, data[0]); } + is_empty = metrics.get().empty(); break; } case constants::CorrectedInt: @@ -218,6 +220,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot type, data[0]); } + is_empty = metrics.get().empty(); break; } case constants::Q: @@ -235,6 +238,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot options, type, data[0]); + is_empty = metrics.get().empty(); break; } case constants::Error://TODO: skip last cycle of read for error metric @@ -247,6 +251,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot options, type, data[0]); + is_empty = metrics.get().empty(); break; } case constants::EmpiricalPhasing: @@ -261,12 +266,13 @@ namespace illumina { namespace interop { namespace logic { namespace plot options, type, data[0]); + is_empty = metrics.get().empty(); break; } default: INTEROP_THROW(model::invalid_metric_type, "Invalid metric group"); } - if(metrics.is_group_empty(logic::utils::to_group(type))) + if(is_empty) { data.clear(); return; diff --git a/src/interop/logic/plot/plot_flowcell_map.cpp b/src/interop/logic/plot/plot_flowcell_map.cpp index fc2292aa8..c0abb944c 100644 --- a/src/interop/logic/plot/plot_flowcell_map.cpp +++ b/src/interop/logic/plot/plot_flowcell_map.cpp @@ -83,7 +83,10 @@ namespace illumina { namespace interop { namespace logic { namespace plot layout.tiles_per_lane()); else { - if(metrics.is_group_empty(logic::utils::to_group(type))) return; + const size_t buffer_size = layout.lane_count()* + layout.total_swaths(layout.surface_count() > 1 && !options.is_specific_surface()) * + layout.tiles_per_lane(); + if(buffer_size == 0) return; data.set_buffer(buffer, tile_buffer, layout.lane_count(), layout.total_swaths(layout.surface_count() > 1 && !options.is_specific_surface()), layout.tiles_per_lane()); @@ -96,6 +99,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot INTEROP_THROW( model::invalid_filter_option, "All cycles is unsupported"); if(utils::is_read_metric(type) && options.all_reads() && metrics.run_info().reads().size() > 1) INTEROP_THROW( model::invalid_filter_option, "All reads is unsupported"); + bool is_empty = true; switch(logic::utils::to_group(type)) { case constants::Tile: @@ -104,6 +108,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot typedef model::metric_base::metric_set metric_set_t; const metric_set_t& metric_set = metrics.get(); metric::metric_value proxy(options.read()); + is_empty = metric_set.empty(); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); break; @@ -117,6 +122,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot if(options.all_channels(type)) INTEROP_THROW(model::invalid_filter_option, "All channels is unsupported"); metric::metric_value proxy(channel); + is_empty = metric_set.empty(); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); break; @@ -130,6 +136,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot if(options.all_bases(type)) INTEROP_THROW( model::invalid_filter_option, "All bases is unsupported"); metric::metric_value proxy(base); + is_empty = metric_set.empty(); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); break; @@ -146,6 +153,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot logic::metric::populate_cumulative_distribution(metric_set); } metric::metric_value proxy; + is_empty = metric_set.empty(); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); break; @@ -156,6 +164,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot typedef model::metric_base::metric_set metric_set_t; const metric_set_t& metric_set = metrics.get(); metric::metric_value proxy; + is_empty = metric_set.empty(); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); break; @@ -166,6 +175,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot typedef model::metric_base::metric_set metric_set_t; const metric_set_t& metric_set = metrics.get(); metric::metric_value proxy; + is_empty = metric_set.empty(); populate_flowcell_map(metric_set.begin(), metric_set.end(), proxy, type, layout, options, data, values_for_scaling); break; @@ -173,7 +183,7 @@ namespace illumina { namespace interop { namespace logic { namespace plot default: INTEROP_THROW( model::invalid_metric_type, "Unsupported metric type: " << constants::to_string(type)); }; - if(metrics.is_group_empty(logic::utils::to_group(type))) + if(is_empty) { data.clear(); return; diff --git a/src/interop/logic/plot/plot_qscore_heatmap.cpp b/src/interop/logic/plot/plot_qscore_heatmap.cpp index fef51e668..96eabc5af 100644 --- a/src/interop/logic/plot/plot_qscore_heatmap.cpp +++ b/src/interop/logic/plot/plot_qscore_heatmap.cpp @@ -187,6 +187,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot */ size_t count_rows_for_heatmap(const model::metrics::run_metrics& metrics) { + if (metrics.get< model::metrics::q_by_lane_metric >().size() > 0) + return metrics.get< model::metrics::q_by_lane_metric >().max_cycle(); return metrics.get< model::metrics::q_metric >().max_cycle(); } /** Count number of columns for the heat map @@ -196,6 +198,8 @@ namespace illumina { namespace interop { namespace logic { namespace plot */ size_t count_columns_for_heatmap(const model::metrics::run_metrics& metrics) { + if (metrics.get< model::metrics::q_by_lane_metric >().size() > 0) + return logic::metric::max_qval(metrics.get< model::metrics::q_by_lane_metric >()); return logic::metric::max_qval(metrics.get< model::metrics::q_metric >()); } diff --git a/src/interop/logic/table/create_imaging_table.cpp b/src/interop/logic/table/create_imaging_table.cpp index e790acef5..875945553 100644 --- a/src/interop/logic/table/create_imaging_table.cpp +++ b/src/interop/logic/table/create_imaging_table.cpp @@ -52,7 +52,7 @@ namespace illumina { namespace interop { namespace logic { namespace table if(data_beg[row*column_count]==0) { if((beg->cycle()-1) >= cycle_to_read.size()) - INTEROP_THROW(model::index_out_of_bounds_exception, "Cycle exceeds total cycles from Reads in the RunInfo.xml"); + INTEROP_THROW(model::index_out_of_bounds_exception, "Cycle exceeds total cycles from Reads in the RunInfo.xml - " << (beg->cycle()-1) << " >= " << cycle_to_read.size()); INTEROP_ASSERTMSG(row_it != row_offset.end(), "Bug with row offset"); @@ -238,7 +238,8 @@ namespace illumina { namespace interop { namespace logic { namespace table 0, naming_method, cmap, - data_beg+row*column_count, data_end); + data_beg+row*column_count, + data_end); } const dynamic_phasing_metric_set_t& dynamic_phasing_metrics = metrics.get(); @@ -376,6 +377,11 @@ namespace illumina { namespace interop { namespace logic { namespace table const constants::metric_group group = to_group(columns[i]); if(group >= constants::MetricCount)continue; valid_to_load[group] = static_cast(1); + if(group == constants::Q) + { + valid_to_load[constants::QCollapsed] = static_cast(1); + valid_to_load[constants::QByLane] = static_cast(1); + } } } diff --git a/src/interop/logic/utils/metrics_to_load.cpp b/src/interop/logic/utils/metrics_to_load.cpp index bbff94e8b..3da70b212 100644 --- a/src/interop/logic/utils/metrics_to_load.cpp +++ b/src/interop/logic/utils/metrics_to_load.cpp @@ -18,8 +18,11 @@ namespace illumina { namespace interop { namespace logic { namespace utils * * @param group specific metric group to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_metrics_to_load(const constants::metric_group group, std::vector& valid_to_load) + void list_metrics_to_load(const constants::metric_group group, + std::vector& valid_to_load, + const constants::instrument_type instrument) { if(valid_to_load.size() != constants::MetricCount) valid_to_load.assign(constants::MetricCount, 0); if(group < constants::MetricCount) @@ -32,7 +35,13 @@ namespace illumina { namespace interop { namespace logic { namespace utils //Hence when Tile is loaded on-demand, we must also load EmpiricalPhasing too if(group == constants::Tile) { - valid_to_load[constants::EmpiricalPhasing] = static_cast(1); + if(instrument == constants::NovaSeq) + valid_to_load[constants::EmpiricalPhasing] = static_cast(1); + } + if(group == constants::Q) + { + valid_to_load[constants::QCollapsed] = static_cast(1); + valid_to_load[constants::QByLane] = static_cast(1); } } @@ -40,45 +49,55 @@ namespace illumina { namespace interop { namespace logic { namespace utils * * @param type specific metric type to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_metrics_to_load(const constants::metric_type type, std::vector& valid_to_load) + void list_metrics_to_load(const constants::metric_type type, + std::vector& valid_to_load, + const constants::instrument_type instrument) { - list_metrics_to_load(utils::to_group(type), valid_to_load); + list_metrics_to_load(utils::to_group(type), valid_to_load, instrument); } /** List the required on demand metrics * * @param groups collection of specific metric groups to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ void list_metrics_to_load(const std::vector& groups, - std::vector& valid_to_load) + std::vector& valid_to_load, + const constants::instrument_type instrument) { for(std::vector::const_iterator it = groups.begin();it != groups.end();++it) - list_metrics_to_load(*it, valid_to_load); + list_metrics_to_load(*it, valid_to_load, instrument); } /** List the required on demand metrics * * @param types collection of specific metric types to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ void list_metrics_to_load(const std::vector& types, - std::vector& valid_to_load) + std::vector& valid_to_load, + const constants::instrument_type instrument) { for(std::vector::const_iterator it = types.begin();it != types.end();++it) - list_metrics_to_load(*it, valid_to_load); + list_metrics_to_load(*it, valid_to_load, instrument); } /** List the required on demand metrics * * @param metric_name name of metric value to load * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_metrics_to_load(const std::string& metric_name, std::vector& valid_to_load) + void list_metrics_to_load(const std::string& metric_name, + std::vector& valid_to_load, + const constants::instrument_type instrument) throw(model::invalid_metric_type) { const constants::metric_type type = constants::parse(metric_name); if(type == constants::UnknownMetricType) INTEROP_THROW(model::invalid_metric_type, "Unsupported metric type: " << metric_name); - list_metrics_to_load(type, valid_to_load); + list_metrics_to_load(type, valid_to_load, instrument); } /** List all required metric groups @@ -99,8 +118,10 @@ namespace illumina { namespace interop { namespace logic { namespace utils /** List all required metric groups * * @param groups destination group list + * @param instrument instrument type */ - void list_summary_metric_groups(std::vector& groups) + void list_summary_metric_groups(std::vector& groups, + const constants::instrument_type instrument) { using namespace model::metrics; groups.clear(); @@ -109,20 +130,25 @@ namespace illumina { namespace interop { namespace logic { namespace utils static_cast(tile_metric::TYPE), static_cast(error_metric::TYPE), static_cast(extraction_metric::TYPE), - static_cast(corrected_intensity_metric::TYPE), - static_cast(phasing_metric::TYPE) + static_cast(corrected_intensity_metric::TYPE) }; groups.assign(group_set, group_set+util::length_of(group_set)); + if(instrument == constants::NovaSeq) + { + groups.push_back(static_cast(phasing_metric::TYPE)); + } } /** List all required metric groups * * @param valid_to_load list of metrics to load on demand + * @param instrument instrument type */ - void list_summary_metrics_to_load(std::vector& valid_to_load) + void list_summary_metrics_to_load(std::vector& valid_to_load, + const constants::instrument_type instrument) { std::vector groups; - list_summary_metric_groups(groups); - logic::utils::list_metrics_to_load(groups, valid_to_load); // Only load the InterOp files required + list_summary_metric_groups(groups, instrument); + logic::utils::list_metrics_to_load(groups, valid_to_load, instrument); // Only load the InterOp files required } /** List all required metric groups @@ -133,7 +159,7 @@ namespace illumina { namespace interop { namespace logic { namespace utils { std::vector groups; list_index_summary_metric_groups(groups); - logic::utils::list_metrics_to_load(groups, valid_to_load); // Only load the InterOp files required + logic::utils::list_metrics_to_load(groups, valid_to_load, constants::UnknownInstrument); // Only load the InterOp files required } /** List all required metric groups for the analysis tab * @@ -145,7 +171,7 @@ namespace illumina { namespace interop { namespace logic { namespace utils description_vector_t types; logic::plot::list_flowcell_metrics(types); for(description_vector_t::const_iterator it = types.begin();it != types.end();++it) - list_metrics_to_load(*it, valid_to_load); + list_metrics_to_load(*it, valid_to_load, constants::UnknownInstrument); } diff --git a/src/interop/model/metrics/image_metric.cpp b/src/interop/model/metrics/image_metric.cpp index 46bee263c..064fb6d99 100644 --- a/src/interop/model/metrics/image_metric.cpp +++ b/src/interop/model/metrics/image_metric.cpp @@ -10,6 +10,7 @@ #include #include #include "interop/model/metrics/image_metric.h" +#include "interop/model/metric_base/metric_set.h" #include "interop/io/format/metric_format_factory.h" #include "interop/io/format/text_format_factory.h" #include "interop/io/format/default_layout.h" @@ -149,6 +150,18 @@ namespace illumina { namespace interop { namespace io { return static_cast(sizeof(record_size_t) + sizeof(version_t)); } + + /** Compute the buffer size + * + * @param metric_set set of metrics + * @return buffer size for entire metric set + */ + static size_t compute_buffer_size(const model::metric_base::metric_set& metric_set) + { + return compute_header_size(metric_set) + compute_size(metric_set)* + metric_set.size()* + metric_set.channel_count(); + } }; /** Image Metric Record Layout Version 2 diff --git a/src/interop/model/metrics/index_metric.cpp b/src/interop/model/metrics/index_metric.cpp index 3fc275588..f0e229036 100644 --- a/src/interop/model/metrics/index_metric.cpp +++ b/src/interop/model/metrics/index_metric.cpp @@ -171,6 +171,29 @@ namespace illumina { namespace interop { namespace io { return static_cast(RECORD_SIZE); } + /** Compute the buffer size + * + * @param metric_set set of metrics + * @return buffer size for entire metric set + */ + static size_t compute_buffer_size(const model::metric_base::metric_set& metric_set) + { + typedef model::metric_base::metric_set::const_iterator const_iterator; + typedef index_metric::index_array_t::const_iterator const_index_iterator; + size_t buffer_size = compute_header_size(metric_set); + for(const_iterator it = metric_set.begin();it != metric_set.end();++it) + { + for (const_index_iterator index_it = it->indices().begin(); index_it != it->indices().end(); ++index_it) + { + buffer_size += sizeof(metric_id_t); + buffer_size += sizeof( ::uint16_t ) + index_it->index_seq().size(); + buffer_size += sizeof(cluster_count_t); + buffer_size += sizeof( ::uint16_t ) + index_it->sample_id().size(); + buffer_size += sizeof( ::uint16_t ) + index_it->sample_proj().size(); + } + } + return buffer_size; + } }; /** Index Metric Record Layout Version 2 @@ -326,6 +349,29 @@ namespace illumina { namespace interop { namespace io { return static_cast(RECORD_SIZE); } + /** Compute the buffer size + * + * @param metric_set set of metrics + * @return buffer size for entire metric set + */ + static size_t compute_buffer_size(const model::metric_base::metric_set& metric_set) + { + typedef model::metric_base::metric_set::const_iterator const_iterator; + typedef index_metric::index_array_t::const_iterator const_index_iterator; + size_t buffer_size = compute_header_size(metric_set); + for(const_iterator it = metric_set.begin();it != metric_set.end();++it) + { + for (const_index_iterator index_it = it->indices().begin(); index_it != it->indices().end(); ++index_it) + { + buffer_size += sizeof(metric_id_t); + buffer_size += sizeof( ::uint16_t ) + index_it->index_seq().size(); + buffer_size += sizeof(cluster_count_t); + buffer_size += sizeof( ::uint16_t ) + index_it->sample_id().size(); + buffer_size += sizeof( ::uint16_t ) + index_it->sample_proj().size(); + } + } + return buffer_size; + } }; #pragma pack() diff --git a/src/interop/model/metrics/tile_metric.cpp b/src/interop/model/metrics/tile_metric.cpp index 1ba3229f6..63167116a 100644 --- a/src/interop/model/metrics/tile_metric.cpp +++ b/src/interop/model/metrics/tile_metric.cpp @@ -34,7 +34,7 @@ namespace illumina { namespace interop { namespace io * 2. Version: 2 */ template<> - struct generic_layout : public default_layout<2, 1 /*Multi record */> + struct generic_layout : public default_layout<2, 1 /* Multi record */> { /** @page tile_v2 Tile Version 2 * @@ -103,7 +103,6 @@ namespace illumina { namespace interop { namespace io std::streamsize count = stream_map(stream, rec); if (stream.fail()) return count; float val = rec.value; - if (val != val) val = 0; // TODO: Remove this after baseline switch (rec.code) { case ControlLane: @@ -161,13 +160,12 @@ namespace illumina { namespace interop { namespace io metric_id.set(metric); bool write_id = false; - // We always write this out, even if it is NaN // We always write out ID for the first record if (!std::isnan(metric.m_cluster_density)) { rec.value = metric.m_cluster_density; rec.code = ClusterDensity; - if (write_id) write_binary(out, metric_id); + if (write_id) write_binary(out, metric_id); // This id is never written write_id = true; write_binary(out, rec); } @@ -224,6 +222,12 @@ namespace illumina { namespace interop { namespace io write_binary(out, rec); } } + if (!write_id) // Write out something so the file is not incomplete + { + rec.value = metric.m_cluster_density; + rec.code = ClusterDensity; + write_binary(out, rec); + } return out.tellp(); } /** Throws an unimplemented error @@ -252,6 +256,35 @@ namespace illumina { namespace interop { namespace io return static_cast(sizeof(record_size_t) + sizeof(version_t)); } + /** Compute the buffer size + * + * @param metric_set set of metrics + * @return buffer size for entire metric set + */ + static size_t compute_buffer_size(const model::metric_base::metric_set& metric_set) + { + typedef tile_metric::read_metric_vector::const_iterator const_read_iterator; + typedef model::metric_base::metric_set::const_iterator const_tile_iterator; + + size_t record_count = 0; + for(const_tile_iterator it = metric_set.begin();it != metric_set.end();++it) + { + if (!std::isnan(it->m_cluster_density)) ++record_count; + if (!std::isnan(it->m_cluster_density_pf)) ++record_count; + if (!std::isnan(it->m_cluster_count)) ++record_count; + if (!std::isnan(it->m_cluster_count_pf)) ++record_count; + for (const_read_iterator rit = it->read_metrics().begin(); rit != it->read_metrics().end(); ++rit) + { + if (!std::isnan(rit->percent_prephasing())) ++record_count; + if (!std::isnan(rit->percent_phasing())) ++record_count; + if (!std::isnan(rit->percent_aligned())) ++record_count; + } + } + + return compute_header_size(metric_set) + compute_size(metric_set)* + record_count; + } + private: static tile_metric::read_metric_vector::iterator get_read(tile_metric &metric, tile_metric::read_metric_type::uint_t read) @@ -388,9 +421,7 @@ namespace illumina { namespace interop { namespace io metric_id.set(metric); std::streamsize count = 0; bool write_id = false; - if (!std::isnan(metric.m_cluster_density) || - !std::isnan(metric.m_cluster_density_pf) || - !std::isnan(metric.m_cluster_count) || + if (!std::isnan(metric.m_cluster_count) || !std::isnan(metric.m_cluster_count_pf)) { const ::uint8_t code = 't'; @@ -408,6 +439,12 @@ namespace illumina { namespace interop { namespace io count += stream_map< ::uint8_t >(stream, code); count += map_stream_read(stream, *beg); } + if(!write_id) // Write out something so the file is not incomplete + { + const ::uint8_t code = 't'; + count += stream_map< ::uint8_t >(stream, code); + count += map_stream_tile(stream, metric); + } return count; } /** Throws an unimplemented error @@ -450,6 +487,27 @@ namespace illumina { namespace interop { namespace io return static_cast(sizeof(::uint8_t) + sizeof(record_size_t) + sizeof(float)); } + /** Compute the buffer size + * + * @param metric_set set of metrics + * @return buffer size for entire metric set + */ + static size_t compute_buffer_size(const model::metric_base::metric_set& metric_set) + { + typedef model::metric_base::metric_set::const_iterator const_tile_iterator; + + size_t record_count = 0; + for(const_tile_iterator it = metric_set.begin();it != metric_set.end();++it) + { + if (!std::isnan(it->m_cluster_count) || !std::isnan(it->m_cluster_count_pf)) + ++record_count; + record_count += it->read_metrics().size(); + } + + return compute_header_size(metric_set) + compute_size(metric_set)* + record_count; + } + private: template static std::streamsize map_stream_read(Stream &stream, Metric &metric) diff --git a/src/interop/model/run/info.cpp b/src/interop/model/run/info.cpp index 17ea556db..368898362 100644 --- a/src/interop/model/run/info.cpp +++ b/src/interop/model/run/info.cpp @@ -14,9 +14,9 @@ #endif #include +#include "interop/util/xml_parser.h" #include "interop/logic/utils/channel.h" // todo remove thiss #include "interop/model/run/info.h" -#include "interop/util/xml_parser.h" #include "interop/io/metric_stream.h" #include "interop/logic/utils/enums.h" #include "interop/logic/metric/tile_metric.h" @@ -27,6 +27,80 @@ using namespace illumina::interop::xml; namespace illumina { namespace interop { namespace model { namespace run { + + /** Read run information from the given XML file + * + * @param filename xml file + */ + void info::write(const std::string &filename)const throw(xml::xml_file_not_found_exception,xml::bad_xml_format_exception) + { + std::ofstream fout(filename.c_str()); + if(!fout.good()) throw xml::xml_file_not_found_exception("Unable to open RunInfo.xml for writing"); + write(fout); + } + + /** String containing xml data + * + * @param out output stream + */ + void info::write(std::ostream& out)const throw(xml::bad_xml_format_exception) + { + xml_document doc; + rapidxml::xml_node<>* run_info = doc.add_node("RunInfo"); + doc.add_attribute(run_info, "Version", m_version); + + rapidxml::xml_node<>* run = doc.add_node(run_info, "Run"); + doc.add_attribute(run, "Id", m_name); + doc.add_attribute(run, "Number", m_run_number); + doc.add_node(run, "Flowcell", m_flowcell.m_barcode); + doc.add_node(run, "Instrument", m_instrument_name); + doc.add_node(run, "Date", m_date); + + // Reads + rapidxml::xml_node<>* reads = doc.add_node(run, "Reads"); + for(const_read_iterator rit = m_reads.begin();rit !=m_reads.end();++rit) + { + rapidxml::xml_node<>* read = doc.add_node(reads, "Read"); + doc.add_attribute(read, "Number", rit->number()); + doc.add_attribute(read, "NumCycles", rit->total_cycles()); + doc.add_attribute(read, "IsIndexedRead", rit->is_index()?"Y":"N"); + } + rapidxml::xml_node<>* flowcell = doc.add_node(run, "FlowcellLayout"); + doc.add_attribute(flowcell, "LaneCount", m_flowcell.lane_count()); + doc.add_attribute(flowcell, "SurfaceCount", m_flowcell.surface_count()); + doc.add_attribute(flowcell, "SwathCount", m_flowcell.swath_count()); + doc.add_attribute(flowcell, "TileCount", m_flowcell.tile_count()); + if(m_version == 4) + { + doc.add_attribute(flowcell, "SectionPerLane", m_flowcell.sections_per_lane()); + doc.add_attribute(flowcell, "LanePerSection", m_flowcell.lanes_per_section()); + } + else + { + if(m_flowcell.sections_per_lane() > 1) throw bad_xml_format_exception("SectionPerLane not supported before Version 4"); + if(m_flowcell.lanes_per_section() > 1) throw bad_xml_format_exception("LanePerSection not supported before Version 4"); + } + rapidxml::xml_node<>* tile_set = doc.add_node(flowcell, "TileSet"); + // TODO: check if supported by version + doc.add_attribute(tile_set, "TileNamingConvention", constants::to_string(m_flowcell.naming_method())); + rapidxml::xml_node<>* tiles = doc.add_node(tile_set, "Tiles"); + const flowcell_layout::str_vector_t& tile_names = m_flowcell.tiles(); + for(size_t i=0;i* image_dims = doc.add_node(run, "ImageDimensions"); + doc.add_attribute(image_dims, "Width", m_image_dim.width()); + doc.add_attribute(image_dims, "Height", m_image_dim.height()); + + // TODO: check if supported by version + rapidxml::xml_node<>* image_channels = doc.add_node(run, "ImageChannels"); + for(str_vector_t::const_iterator it = m_channels.begin(); it != m_channels.end();++it) + { + doc.add_node(image_channels, "Name", *it); + } + out << doc; + } + void info::parse(char *data) throw(xml::xml_file_not_found_exception, xml::bad_xml_format_exception, xml::empty_xml_format_exception, @@ -46,7 +120,7 @@ namespace illumina { namespace interop { namespace model { namespace run xml_node_ptr p_root = doc.first_node(); if (p_root == 0) INTEROP_THROW(empty_xml_format_exception, "Root node not found"); if (p_root->name() != std::string("RunInfo")) - INTEROP_THROW(bad_xml_format_exception, "Invalid run info xml file"); + INTEROP_THROW(bad_xml_format_exception, "Invalid run info xml file, note v1 (GA) is not supported"); for (xml_attr_ptr attr = p_root->first_attribute(); attr; attr = attr->next_attribute()) @@ -69,7 +143,8 @@ namespace illumina { namespace interop { namespace model { namespace run for (xml_attr_ptr attr = p_run_node->first_attribute(); attr; attr = attr->next_attribute()) { - if(set_data(attr, "Id", m_name)) break; + if(set_data(attr, "Id", m_name)) continue; + if(set_data(attr, "Number", m_run_number)) continue; } // Parse run data @@ -77,6 +152,7 @@ namespace illumina { namespace interop { namespace model { namespace run { if (set_data(p_node, "Date", m_date)) continue; if (set_data(p_node, "Flowcell", m_flowcell.m_barcode)) continue; + if (set_data(p_node, "Instrument", m_instrument_name)) continue; if (p_node->name() == std::string("FlowcellLayout")) { for (xml_attr_ptr attr = p_node->first_attribute(); @@ -223,7 +299,7 @@ namespace illumina { namespace interop { namespace model { namespace run INTEROP_THROW(invalid_run_info_exception, "Tile number exceeds number of tiles"); const ::uint32_t surface = logic::metric::surface(tile, m_flowcell.naming_method()); if(surface > m_flowcell.surface_count()) - INTEROP_THROW(invalid_run_info_exception, "Surface number exceeds number of surfaces"); + INTEROP_THROW(invalid_run_info_exception, "Surface number exceeds number of surfaces - " << surface << " > " << m_flowcell.surface_count()); const ::uint32_t section = logic::metric::section(tile, m_flowcell.naming_method()); if(m_flowcell.m_naming_method == constants::FiveDigit && section > m_flowcell.total_number_of_sections()) INTEROP_THROW(invalid_run_info_exception, "Section number exceeds number of sections"); diff --git a/src/interop/model/run_metrics.cpp b/src/interop/model/run_metrics.cpp index 7ef2aeec0..e6d050a85 100644 --- a/src/interop/model/run_metrics.cpp +++ b/src/interop/model/run_metrics.cpp @@ -5,6 +5,10 @@ * @version 1.0 * @copyright GNU Public License. */ +#ifdef _OPENMP +#include +#endif + #include "interop/model/run_metrics.h" #include "interop/logic/metric/q_metric.h" @@ -345,11 +349,109 @@ namespace illumina { namespace interop { namespace model { namespace metrics bool_pointer m_load_metric_check; }; + class read_metric_set_from_binary_buffer + { + public: + read_metric_set_from_binary_buffer(const constants::metric_group group, + uint8_t* buffer, + const size_t buffer_size) : + m_group(group), + m_buffer(buffer), + m_buffer_size(buffer_size){} + template + void operator()(MetricSet &metrics) const + { + if(m_group == static_cast(MetricSet::TYPE)) + { + io::read_interop_from_buffer(m_buffer, m_buffer_size, metrics); + } + } + private: + constants::metric_group m_group; + uint8_t* m_buffer; + size_t m_buffer_size; + }; + + class write_metric_set_to_binary_buffer + { + public: + write_metric_set_to_binary_buffer(const constants::metric_group group, + uint8_t* buffer, + const size_t buffer_size) : + m_group(group), + m_buffer(buffer), + m_buffer_size(buffer_size){} + template + void operator()(const MetricSet &metrics) const + { + if(m_group == static_cast(MetricSet::TYPE)) + { + io::write_interop_to_buffer(metrics, m_buffer, m_buffer_size); + } + } + + private: + constants::metric_group m_group; + uint8_t* m_buffer; + size_t m_buffer_size; + }; + class calculate_metric_set_buffer_size + { + public: + calculate_metric_set_buffer_size(const constants::metric_group group) : + m_group(group), m_buffer_size(0){} + template + void operator()(const MetricSet &metrics) + { + if(m_group == static_cast(MetricSet::TYPE)) + { + m_buffer_size = io::compute_buffer_size(metrics); + } + } + size_t buffer_size()const + { + return m_buffer_size; + } + + private: + constants::metric_group m_group; + size_t m_buffer_size; + }; + class list_interop_filenames + { + public: + list_interop_filenames(const constants::metric_group group, + std::vector& files, + const std::string& run_folder, + const size_t last_cycle) : + m_group(group), + m_files(files), + m_run_folder(run_folder), + m_last_cycle(last_cycle) + {} + template + void operator()(const MetricSet &) const + { + if(m_group == static_cast(MetricSet::TYPE)) + { + io::list_interop_filenames< MetricSet >(m_files, m_run_folder, m_last_cycle); + } + } + + private: + constants::metric_group m_group; + std::vector& m_files; + std::string m_run_folder; + size_t m_last_cycle; + }; + + /** Read binary metrics and XML files from the run folder * * @param run_folder run folder path + * @param thread_count number of threads to use for network loading */ - void run_metrics::read(const std::string &run_folder) + void run_metrics::read(const std::string &run_folder, const size_t thread_count) throw(xml::xml_file_not_found_exception, xml::bad_xml_format_exception, xml::empty_xml_format_exception, @@ -365,7 +467,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics { clear(); const size_t count = read_xml(run_folder); - read_metrics(run_folder, run_info().total_cycles()); + read_metrics(run_folder, run_info().total_cycles(), thread_count); finalize_after_load(count); } /** Read binary metrics and XML files from the run folder @@ -373,10 +475,12 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @note This function does not clear * @param run_folder run folder path * @param valid_to_load list of metrics to load + * @param thread_count number of threads to use for network loading * @param skip_loaded skip metrics that are already loaded */ void run_metrics::read(const std::string &run_folder, const std::vector& valid_to_load, + const size_t thread_count, const bool skip_loaded) throw(xml::xml_file_not_found_exception, xml::bad_xml_format_exception, @@ -393,7 +497,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics invalid_parameter) { read_run_info(run_folder); - read_metrics(run_folder, run_info().total_cycles(), valid_to_load, skip_loaded); + read_metrics(run_folder, run_info().total_cycles(), valid_to_load, thread_count, skip_loaded); const size_t count = read_run_parameters(run_folder); finalize_after_load(count); check_for_data_sources(run_folder, run_info().total_cycles()); @@ -432,8 +536,9 @@ namespace illumina { namespace interop { namespace model { namespace metrics /** Read RunParameters.xml if necessary * * @param run_folder run folder path + * @param force_load force loading of run parameters */ - size_t run_metrics::read_run_parameters(const std::string &run_folder) throw(io::file_not_found_exception, + size_t run_metrics::read_run_parameters(const std::string &run_folder, const bool force_load) throw(io::file_not_found_exception, xml::xml_file_not_found_exception, xml::bad_xml_format_exception, xml::empty_xml_format_exception, @@ -441,7 +546,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics xml::xml_parse_exception) { const size_t count = count_legacy_bins(); - if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count)) + if (m_run_info.channels().empty() || logic::metric::requires_legacy_bins(count) || force_load) { try { @@ -452,7 +557,7 @@ namespace illumina { namespace interop { namespace model { namespace metrics if (m_run_info.channels().empty()) INTEROP_THROW(io::file_not_found_exception, "RunParameters.xml required for legacy run folders with missing channel names"); - else + else if(logic::metric::requires_legacy_bins(count)) INTEROP_THROW(io::file_not_found_exception, "RunParameters.xml required for legacy run folders and is missing"); } @@ -621,18 +726,31 @@ namespace illumina { namespace interop { namespace model { namespace metrics * * @param run_folder run folder path * @param last_cycle last cycle to search for by cycle interops + * @param thread_count number of threads to use for network loading */ - void run_metrics::read_metrics(const std::string &run_folder, const size_t last_cycle) throw( + void run_metrics::read_metrics(const std::string &run_folder, const size_t last_cycle, const size_t thread_count) + throw( io::file_not_found_exception, io::bad_format_exception, io::incomplete_file_exception) { - read_func read_functor(run_folder); - m_metrics.apply(read_functor); - if (read_functor.are_all_files_missing()) +#ifdef _OPENMP + if(thread_count > 1) { - m_metrics.apply(read_by_cycle_func(run_folder, last_cycle)); + std::vector valid_to_load(constants::MetricCount, 1); + read_metrics(run_folder, last_cycle, valid_to_load, thread_count); + } + else{ +#endif + read_func read_functor(run_folder); + m_metrics.apply(read_functor); + if (read_functor.are_all_files_missing()) + { + m_metrics.apply(read_by_cycle_func(run_folder, last_cycle)); + } +#ifdef _OPENMP } +#endif } /** Read binary metrics from the run folder @@ -645,11 +763,13 @@ namespace illumina { namespace interop { namespace model { namespace metrics * @param run_folder run folder path * @param last_cycle last cycle to search for by cycle interops * @param valid_to_load list of metrics to load + * @param thread_count number of threads to use for network loading * @param skip_loaded skip metrics that are already loaded */ void run_metrics::read_metrics(const std::string &run_folder, const size_t last_cycle, const std::vector& valid_to_load, + const size_t thread_count, const bool skip_loaded) throw(io::file_not_found_exception, io::bad_format_exception, @@ -661,11 +781,92 @@ namespace illumina { namespace interop { namespace model { namespace metrics INTEROP_THROW(invalid_parameter, "Boolean array valid_to_load does not match expected number of metrics: " << valid_to_load.size() << " != " << constants::MetricCount); - read_func read_functor(run_folder, &valid_to_load.front(), skip_loaded); - m_metrics.apply(read_functor); - if (read_functor.are_all_files_missing()) - { - m_metrics.apply(read_by_cycle_func(run_folder, last_cycle, &valid_to_load.front())); + bool all_files_are_missing = true; +#ifdef _OPENMP + if(thread_count > 1) + { + std::vector local_files_missing(thread_count, true); + std::vector offset; + offset.reserve(valid_to_load.size()); + for(size_t i=0;i > valid_to_load_local(thread_count, std::vector(valid_to_load.size(), 0)); + bool exception_thrown = false; + std::string exception_msg; +# pragma omp parallel for default(shared) num_threads(static_cast(thread_count)) schedule(dynamic) + for(int i=0;i(offset.size());++i) + { +# pragma omp flush(exception_thrown) + if(exception_thrown) continue; + valid_to_load_local[ omp_get_thread_num() ][offset[i]] = 1; + read_func read_functor_l(run_folder, &valid_to_load_local[ omp_get_thread_num() ].front(), skip_loaded); + try{ + m_metrics.apply(read_functor_l); + } + catch(const std::exception& ex) + { +#pragma omp critical(SaveMessage) + exception_msg = ex.what(); + + exception_thrown = true; +#pragma omp flush(exception_thrown) + } + valid_to_load_local[ omp_get_thread_num() ][offset[i]] = 0; + local_files_missing[omp_get_thread_num()] = local_files_missing[omp_get_thread_num()] && read_functor_l.are_all_files_missing(); + } + if(exception_thrown) + throw io::bad_format_exception(exception_msg); + for(size_t i=0;i 1) + { + std::vector offset; + offset.reserve(valid_to_load.size()); + for(size_t i=0;i > valid_to_load_local(thread_count, std::vector(valid_to_load.size(), 0)); + bool exception_thrown = false; + std::string exception_msg; +# pragma omp parallel for default(shared) num_threads(static_cast(thread_count)) schedule(dynamic) + for(int i=0;i(offset.size());++i) + { +# pragma omp flush(exception_thrown) + valid_to_load_local[ omp_get_thread_num() ][offset[i]] = 1; + try{ + m_metrics.apply(read_by_cycle_func(run_folder, last_cycle, &valid_to_load_local[ omp_get_thread_num() ].front())); + } + catch(const std::exception& ex) + { +#pragma omp critical(SaveMessage) + exception_msg = ex.what(); + + exception_thrown = true; +#pragma omp flush(exception_thrown) + } + valid_to_load_local[ omp_get_thread_num() ][offset[i]] = 0; + } + if(exception_thrown) + throw io::bad_format_exception(exception_msg); + } + else + { +#endif + m_metrics.apply(read_by_cycle_func(run_folder, last_cycle, &valid_to_load.front())); +#ifdef _OPENMP + } +#endif } } @@ -680,6 +881,66 @@ namespace illumina { namespace interop { namespace model { namespace metrics m_metrics.apply(write_func(run_folder)); } + /** Read a single metric set from a binary buffer + * + * @param group metric set to write + * @param buffer binary buffer + * @param buffer_size size of binary buffer + */ + void run_metrics::read_metrics_from_buffer(const constants::metric_group group, + uint8_t* buffer, + const size_t buffer_size) throw( + io::file_not_found_exception, + io::bad_format_exception, + io::incomplete_file_exception, + model::index_out_of_bounds_exception) + { + m_metrics.apply(read_metric_set_from_binary_buffer(group, buffer, buffer_size)); + } + /** Write a single metric set to a binary buffer + * + * @param group metric set to write + * @param buffer binary buffer + * @param buffer_size size of binary buffer + */ + void run_metrics::write_metrics_to_buffer(const constants::metric_group group, + uint8_t* buffer, + const size_t buffer_size)const throw( + io::invalid_argument, + io::bad_format_exception, + io::incomplete_file_exception) + { + m_metrics.apply(write_metric_set_to_binary_buffer(group, buffer, buffer_size)); + } + + /** List all filenames for a specific metric + * + * @param group metric group type + * @param files destination interop file names (first one is legacy, all subsequent are by cycle) + * @param run_folder run folder location + */ + void run_metrics::list_filenames(const constants::metric_group group, + std::vector& files, + const std::string& run_folder) + throw(invalid_run_info_exception) + { + const size_t last_cycle = run_info().total_cycles(); + if( last_cycle == 0 ) INTEROP_THROW(invalid_run_info_exception, "RunInfo is empty"); + m_metrics.apply(list_interop_filenames(group, files, run_folder, last_cycle)); + } + /** Calculate the required size of the buffer for writing + * + * @param group metric set to write + * @return required size of the binary buffer + */ + size_t run_metrics::calculate_buffer_size(const constants::metric_group group)const throw( + io::invalid_argument, io::bad_format_exception) + { + calculate_metric_set_buffer_size calc(group); + m_metrics.apply(calc); + return calc.buffer_size(); + } + /** Populate a map of valid tiles * * @param map mapping between tile has and base_metric diff --git a/src/tests/csharp/CoreTests.cs b/src/tests/csharp/CoreTests.cs index 27fa75adc..299263f8c 100644 --- a/src/tests/csharp/CoreTests.cs +++ b/src/tests/csharp/CoreTests.cs @@ -17,6 +17,30 @@ namespace Illumina.InterOp.Interop.UnitTest [TestFixture] public class CoreTests { + /// + /// Test Enum parsing + /// + [Test] + public void ParseMetricTypeEnum() + { + Assert.AreEqual(c_csharp_run.parse_metric_type("Intensity"), metric_type.Intensity); + } + /// + /// Test Enum parsing + /// + [Test] + public void ToStringMetricTypeEnum() + { + Assert.AreEqual(c_csharp_run.to_string(metric_type.Intensity), "Intensity"); + } + /// + /// Test Enum parsing + /// + [Test] + public void ToStringMetricGroupEnum() + { + Assert.AreEqual(c_csharp_run.to_string(metric_group.Error), "Error"); + } /// /// Test IndexOutOfBoundsException /// @@ -202,6 +226,42 @@ public void TestDispose() Assert.AreEqual((int)set.at(0).lane(), 1); } } + [Test] + public void TestReadFromBinaryBuffer() + { + int[] tmp = new int[]{2, 48, 1, 0, 80, 4, 25, 0, 39, 4, 189, 4, 198, 3, 192, 3, 71, 4, 230, 15, 234, 15, 189, 15, 132, + 15, 0, 0, 0, 0, 65, 168, 10, 0, 93, 93, 8, 0, 104, 95, 8, 0, 238, 221, 9, 0, 91, 34, 63, 65, 1, 0, + 80, 4, 1, 0, 15, 5, 22, 6, 127, 4, 134, 4, 13, 5, 149, 19, 119, 19, 51, 19, 68, 19, 186, 42, 0, 0, + 221, 49, 11, 0, 101, 53, 8, 0, 168, 76, 8, 0, 80, 100, 9, 0, 5, 226, 84, 65, 1, 0, 81, 4, 25, 0, 1, + 4, 147, 4, 164, 3, 144, 3, 45, 4, 91, 15, 91, 15, 83, 15, 38, 15, 0, 0, 0, 0, 171, 201, 10, 0, 153, + 125, 8, 0, 35, 124, 8, 0, 135, 250, 9, 0, 130, 213, 59, 65 + }; + byte[] expected_binary_data = new byte[tmp.Length]; + for(int i=0;i 1 && !options.is_specific_surface()); - uint tile_count = layout.tiles_per_lane(); - float[] data_buffer = new float[lane_count*swath_count*tile_count]; - uint[] tile_buffer = new uint[lane_count*swath_count*tile_count]; + uint flowcell_size = c_csharp_plot.calculate_flowcell_buffer_size(run, options); + float[] data_buffer = new float[flowcell_size]; + uint[] tile_buffer = new uint[flowcell_size]; flowcell_data data = new flowcell_data(); - c_csharp_plot.plot_flowcell_map(run, metric_type.QScore, options, data, data_buffer, tile_buffer); - Assert.AreEqual(data.row_count(), 8); + c_csharp_plot.plot_flowcell_map(run, metric_type.Q20Percent, options, data, data_buffer, tile_buffer); + Assert.AreEqual(1152, flowcell_size); + Assert.AreEqual(8, data.row_count()); } /// /// Test bad metric name exception @@ -81,12 +75,7 @@ public void TestBadMetricException() reads.Add(new read_info(1, 1, 26)); reads.Add(new read_info(2, 27, 76)); run.run_info(new info( - "", - "", - 1, - new flowcell_layout(8, 2, 2, 36, 1, 1, new string_vector(), tile_naming_method.FourDigit), - new string_vector(), - new image_dimensions(), + new flowcell_layout(8, 2, 2, 36, 1, 1, new string_vector(), tile_naming_method.FourDigit), reads )); run.legacy_channel_update(instrument_type.HiSeq); diff --git a/src/tests/csharp/logic/PlotQScoreHeatmap.cs b/src/tests/csharp/logic/PlotQScoreHeatmap.cs index d488b661f..fb390ca1b 100644 --- a/src/tests/csharp/logic/PlotQScoreHeatmap.cs +++ b/src/tests/csharp/logic/PlotQScoreHeatmap.cs @@ -37,12 +37,7 @@ public void Heatmap() reads.Add(new read_info(1, 1, 26)); reads.Add(new read_info(2, 27, 76)); run.run_info(new info( - "", - "", - 1, - new flowcell_layout(8, 2, 2, 36, 1, 1, new string_vector(), tile_naming_method.FourDigit), - new string_vector(), - new image_dimensions(), + new flowcell_layout(8, 2, 2, 36, 1, 1, new string_vector(), tile_naming_method.FourDigit), reads )); run.legacy_channel_update(instrument_type.HiSeq); diff --git a/src/tests/csharp/logic/PlotQScoreHistogram.cs b/src/tests/csharp/logic/PlotQScoreHistogram.cs index c25203dd7..3111eea30 100644 --- a/src/tests/csharp/logic/PlotQScoreHistogram.cs +++ b/src/tests/csharp/logic/PlotQScoreHistogram.cs @@ -38,12 +38,7 @@ public void Histogram() reads.Add(new read_info(1, 1, 26)); reads.Add(new read_info(2, 27, 76)); run.run_info(new info( - "", - "", - 1, - new flowcell_layout(8, 2, 2, 36, 1, 1, new string_vector(), tile_naming_method.FourDigit), - new string_vector(), - new image_dimensions(), + new flowcell_layout(8, 2, 2, 36, 1, 1, new string_vector(), tile_naming_method.FourDigit), reads )); run.legacy_channel_update(instrument_type.HiSeq); diff --git a/src/tests/csharp/logic/PlotSampleQC.cs b/src/tests/csharp/logic/PlotSampleQC.cs index c06231ef6..c2e860e2e 100644 --- a/src/tests/csharp/logic/PlotSampleQC.cs +++ b/src/tests/csharp/logic/PlotSampleQC.cs @@ -68,12 +68,7 @@ public void ReadsIdentified() reads.Add(new read_info(1, 1, 26)); reads.Add(new read_info(2, 27, 76)); run.run_info(new info( - "", - "", - 1, - new flowcell_layout(2, 2, 2, 16), - new string_vector(), - new image_dimensions(), + new flowcell_layout(2, 2, 2, 16), reads )); run.legacy_channel_update(instrument_type.HiSeq); diff --git a/src/tests/csharp/metrics/RunMetricsTest.cs b/src/tests/csharp/metrics/RunMetricsTest.cs index 83e8121f2..d365713ae 100644 --- a/src/tests/csharp/metrics/RunMetricsTest.cs +++ b/src/tests/csharp/metrics/RunMetricsTest.cs @@ -24,20 +24,14 @@ public void TestListErrorMetricFilenames() read_info_vector reads = new read_info_vector(); reads.Add(new read_info(1, 1, 3)); - run.run_info(new info( - "", - "", - 1, - new flowcell_layout(2, 2, 2, 16), - new string_vector(), - new image_dimensions(), + run.run_info(new info(new flowcell_layout(2, 2, 2, 16), reads )); run.set_naming_method(tile_naming_method.FourDigit); run.legacy_channel_update(instrument_type.HiSeq); string_vector filenames = new string_vector(); - run.list_error_metric_filenames(filenames, "RunFolder"); + run.list_filenames(metric_group.Error, filenames, "RunFolder"); Assert.AreEqual(filenames.Count, 4); Assert.AreEqual(filenames[0], Path.Combine("RunFolder", "InterOp", "ErrorMetricsOut.bin")); Assert.AreEqual(filenames[1], Path.Combine("RunFolder", "InterOp", "C1.1", "ErrorMetricsOut.bin")); diff --git a/src/tests/csharp/run/RunInfoTest.cs b/src/tests/csharp/run/RunInfoTest.cs index c6a7ec541..357cab809 100644 --- a/src/tests/csharp/run/RunInfoTest.cs +++ b/src/tests/csharp/run/RunInfoTest.cs @@ -122,10 +122,14 @@ protected override void SetUp() string Run_Id = "120705_M00903_0009_A000000000-A12V4"; string Date = "120705"; + string InstrumentName = "M00903"; + uint RunNumber = 8; uint version = 2; image_dimensions ImageDimensions = new image_dimensions(/*Width*/ 0, /*Height*/ 0); expected_run_info = new info(Run_Id, Date, + InstrumentName, + RunNumber, version, FlowcellLayout, ImageChannels, diff --git a/src/tests/interop/CMakeLists.txt b/src/tests/interop/CMakeLists.txt index 034269cb3..575bc4bcc 100644 --- a/src/tests/interop/CMakeLists.txt +++ b/src/tests/interop/CMakeLists.txt @@ -37,9 +37,10 @@ set(SRCS logic/index_summary_test.cpp metrics/coverage_test.cpp metrics/metric_stream_error_test.cpp - ) + metrics/metric_regression_tests.cpp) set(HEADERS + logic/inc/collapsed_q_plot_test_generator.h metrics/inc/corrected_intensity_metrics_test.h metrics/inc/error_metrics_test.h metrics/inc/metric_test.h @@ -62,6 +63,7 @@ set(HEADERS logic/inc/empty_plot_test_generator.h logic/inc/plot_regression_test_generator.h metrics/inc/format_registry.h + run/info_test.h ) diff --git a/src/tests/interop/inc/abstract_regression_test_generator.h b/src/tests/interop/inc/abstract_regression_test_generator.h index 069ed34f4..32c83741b 100644 --- a/src/tests/interop/inc/abstract_regression_test_generator.h +++ b/src/tests/interop/inc/abstract_regression_test_generator.h @@ -73,7 +73,6 @@ namespace illumina{ namespace interop { namespace unittest { const regression_test_data& data = regression_test_data::instance(); const std::string baseline_file = baseline(); - ::testing::Message msg; if(!data.rebaseline()) { const bool expected_found = read_expected(baseline_file, expected); @@ -93,14 +92,9 @@ namespace illumina{ namespace interop { namespace unittest { if(generate_actual(m_run_folder, actual)) { - msg << "[ ] Rebaseline: " << *this; if(!write_actual(baseline_file, actual)) return ::testing::AssertionFailure() << "Failed to write baseline: " << baseline_file; } - else - { - msg << "[ ] Skipped: " << *this; - } *skip_test = true; } catch(const std::exception& ex) @@ -110,7 +104,7 @@ namespace illumina{ namespace interop { namespace unittest //return ::testing::AssertionFailure() << "Failed to generate baseline: " << baseline_file << " " << ex.what(); } } - return ::testing::AssertionFailure(msg); + return ::testing::AssertionSuccess(); } /** Get the full path of the baseline output file * diff --git a/src/tests/interop/logic/imaging_table_logic_test.cpp b/src/tests/interop/logic/imaging_table_logic_test.cpp index 56a5fe2e9..36288f4cf 100644 --- a/src/tests/interop/logic/imaging_table_logic_test.cpp +++ b/src/tests/interop/logic/imaging_table_logic_test.cpp @@ -12,6 +12,8 @@ #include "interop/logic/table/create_imaging_table.h" #include "interop/io/table/imaging_table_csv.h" #include "src/tests/interop/metrics/inc/error_metrics_test.h" +#include "src/tests/interop/logic/inc/plot_regression_test_generator.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop; using namespace illumina::interop::unittest; @@ -41,20 +43,9 @@ namespace illumina{ namespace interop {namespace model {namespace table */ void simulate_read_error_metrics(model::metrics::run_metrics& metrics) { - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(2, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::error_metric_v3::create_expected(metrics.get()); } diff --git a/src/tests/interop/logic/inc/collapsed_q_plot_test_generator.h b/src/tests/interop/logic/inc/collapsed_q_plot_test_generator.h new file mode 100644 index 000000000..fc6f38896 --- /dev/null +++ b/src/tests/interop/logic/inc/collapsed_q_plot_test_generator.h @@ -0,0 +1,104 @@ +/** Generate plots for unit testing + * + * + * @file + * @date 11/3/16 + * @version 1.0 + * @copyright GNU Public License. + */ +#pragma once +#include +#include "src/tests/interop/logic/inc/metric_filter_iterator.h" +#include "src/tests/interop/inc/generic_fixture.h" +#include "src/tests/interop/metrics/inc/q_metrics_test.h" +#include "src/tests/interop/run/info_test.h" + +namespace illumina { namespace interop { namespace unittest +{ + /** Generate the plot from an empty metric set + * + * The expected plot is empty + */ + template + class collapsed_q_plot_test_generator : public abstract_generator + { + typedef typename abstract_generator::parent_type base_type; + public: + /** Constructor + * + * @param plot_type type of the plot + */ + collapsed_q_plot_test_generator(const constants::plot_types plot_type=constants::UnknownPlotType) : + m_plot_type(plot_type) + { + const model::run::read_info read_array[]={ + model::run::read_info(1, 1, 26), + model::run::read_info(2, 27, 76) + }; + hiseq4k_run_info::create_expected(m_info, util::to_vector(read_array)); + if(plot_type != constants::UnknownPlotType) m_metric_iterator.reset(m_info, plot_type); + } + /** Clone the concrete implementation TODO: Remove + * + * @param plot_type type of plot + * @return copy of this object + */ + virtual base_type operator()(const constants::plot_types plot_type)const + { + return new collapsed_q_plot_test_generator(plot_type); + } + + /** Generate the expected and actual metric sets + * + * @note expected plot data is empty + * @param expected expected plot data + * @param actual actual plot data + */ + virtual ::testing::AssertionResult generate(PlotData& expected, PlotData &actual, bool*) const + { + model::metrics::run_metrics metrics(m_info); + unittest::q_metric_v6::create_expected(metrics.get()); + metrics.finalize_after_load(); + m_metric_iterator.plot(metrics, expected); + + metrics.get().clear(); + m_metric_iterator.plot(metrics, actual); + return ::testing::AssertionSuccess(); + } + + /** Create a copy of this object + * + * @return pointer to an abstract_generator + */ + virtual base_type clone() const + { + return new collapsed_q_plot_test_generator(*this); + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << constants::to_string(m_plot_type); + out << "Metric_" << constants::to_string(m_metric_iterator.metric()) << "_"; + out << m_metric_iterator.options(); + } + /** Advance to the next type + * + * @return true when the generator has finished, and the next parameter can be obtained + */ + bool advance() + { + return m_metric_iterator.advance(); + } + + protected: + metric_filter_iterator m_metric_iterator; + model::run::info m_info; + constants::plot_types m_plot_type; + }; + +}}} + diff --git a/src/tests/interop/logic/inc/plot_regression_test_generator.h b/src/tests/interop/logic/inc/plot_regression_test_generator.h index 0e24057fd..50ec9a19e 100644 --- a/src/tests/interop/logic/inc/plot_regression_test_generator.h +++ b/src/tests/interop/logic/inc/plot_regression_test_generator.h @@ -67,7 +67,7 @@ namespace illumina { namespace interop { namespace unittest bool generate_actual(const std::string& run_folder, PlotData& actual)const { model::metrics::run_metrics& actual_metrics = get_metrics(run_folder); - if( actual_metrics.empty() ) return false; + if( actual_metrics.empty() )return false; try { m_metric_iterator.plot(actual_metrics, actual); @@ -163,7 +163,6 @@ namespace illumina { namespace interop { namespace unittest std::string name()const { return constants::to_string(m_plot_type) + "_" - + io::basename(parent_t::m_run_folder) + "_Metric_" + constants::to_string(m_metric_iterator.metric()) + "_" + util::lexical_cast(m_metric_iterator.options()); } diff --git a/src/tests/interop/logic/index_summary_test.cpp b/src/tests/interop/logic/index_summary_test.cpp index 5296aa9e1..1d165657e 100644 --- a/src/tests/interop/logic/index_summary_test.cpp +++ b/src/tests/interop/logic/index_summary_test.cpp @@ -11,6 +11,7 @@ #include "src/tests/interop/metrics/inc/index_metrics_test.h" #include "src/tests/interop/metrics/inc/tile_metrics_test.h" #include "src/tests/interop/inc/abstract_regression_test_generator.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop::model::summary; @@ -68,18 +69,8 @@ TEST_P(index_summary_tests, index_lane_summary) /** TODO take tile metrics and index metrics from the same run */ TEST(index_summary_test, lane_summary) { - const size_t lane_count = 8; - std::vector reads(1, model::run::read_info(1, 1, 3, false)); - std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count), - channels, - model::run::image_dimensions(), - reads); + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); model::summary::index_flowcell_summary expected; index_metric_v1::create_summary(expected); @@ -176,36 +167,15 @@ class index_summary_generator : public abstract_generator channels; - channels.push_back("Red"); - channels.push_back("Green"); - - std::vector reads; - reads.reserve(3); - reads.push_back(model::run::read_info(1, 1, 3, false)); - reads.push_back(model::run::read_info(2, 4, 10, false)); - reads.push_back(model::run::read_info(3, 11, 15, false)); - actual = model::summary::index_flowcell_summary(); - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count, - surface_count, - swath_count, - tile_count, - sections_per_lane, - lanes_per_section), - channels, - model::run::image_dimensions(), - reads); - run_info.set_naming_method(constants::FourDigit); + + model::run::info run_info; + const model::run::read_info read_array[]={ + model::run::read_info(1, 1, 3, false), + model::run::read_info(2, 4, 10, false), + model::run::read_info(3, 11, 15, false) + }; + hiseq4k_run_info::create_expected(run_info, util::to_vector(read_array)); model::metrics::run_metrics metrics(run_info); IndexGen::create_expected(metrics.get()); TileGen::create_expected(metrics.get()); diff --git a/src/tests/interop/logic/plot_candle_stick_test.cpp b/src/tests/interop/logic/plot_candle_stick_test.cpp index 9adee611a..2297229b1 100644 --- a/src/tests/interop/logic/plot_candle_stick_test.cpp +++ b/src/tests/interop/logic/plot_candle_stick_test.cpp @@ -15,7 +15,9 @@ #include "src/tests/interop/metrics/inc/error_metrics_test.h" #include "src/tests/interop/metrics/inc/extraction_metrics_test.h" #include "src/tests/interop/logic/inc/empty_plot_test_generator.h" +#include "src/tests/interop/logic/inc/collapsed_q_plot_test_generator.h" #include "src/tests/interop/logic/inc/plot_regression_test_generator.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::plot; @@ -31,10 +33,12 @@ struct candle_stick_tests : public generic_test_fixture /** Test that the filter iterator works */ TEST(candle_stick_tests, test_filter_iterator_by_cycle) { - const std::string channels[] = {"Red", "Green"}; - const model::run::read_info reads[] = {model::run::read_info(1, 3, false)}; - model::run::info run_info(model::run::flowcell_layout(2, 2), util::to_vector(channels), util::to_vector(reads)); - run_info.set_naming_method(constants::FourDigit); + + model::run::info run_info; + const model::run::read_info read_array[]={ + model::run::read_info(1, 1, 4, false) + }; + hiseq4k_run_info::create_expected(run_info, util::to_vector(read_array)); metric_filter_iterator metric_iterator; metric_iterator.reset(run_info, constants::ByCyclePlot); while(!metric_iterator.is_done()) @@ -99,6 +103,12 @@ INSTANTIATE_TEST_CASE_P(candle_stick_unit_test, candle_stick_tests, ProxyValuesIn(plot_by_cycle_gen, plot_by_cycle_gen_data)); + +collapsed_q_plot_test_generator collapsed_plot_by_cycle_gen; +INSTANTIATE_TEST_CASE_P(collapsed_candle_stick_unit_test, + candle_stick_tests, + ProxyValuesIn(collapsed_plot_by_cycle_gen, plot_by_cycle_gen_data)); + class candle_stick_read_generator { public: diff --git a/src/tests/interop/logic/plot_flowcell_test.cpp b/src/tests/interop/logic/plot_flowcell_test.cpp index 8290b7bb6..f318694ae 100644 --- a/src/tests/interop/logic/plot_flowcell_test.cpp +++ b/src/tests/interop/logic/plot_flowcell_test.cpp @@ -12,6 +12,7 @@ #include "interop/model/plot/flowcell_data.h" #include "interop/model/plot/filter_options.h" #include "src/tests/interop/logic/inc/empty_plot_test_generator.h" +#include "src/tests/interop/logic/inc/collapsed_q_plot_test_generator.h" #include "src/tests/interop/logic/inc/plot_regression_test_generator.h" @@ -63,6 +64,12 @@ INSTANTIATE_TEST_CASE_P(plot_flowcell_unit_test, flowcell_plot_tests, ProxyValuesIn(plot_flowcell_gen, flowcell_gen_data)); + +collapsed_q_plot_test_generator plot_collpased_flowcell_gen; +INSTANTIATE_TEST_CASE_P(plot_flowcell_collapsed_unit_test, + flowcell_plot_tests, + ProxyValuesIn(plot_collpased_flowcell_gen, flowcell_gen_data)); + class flowcell_write_read_generator { public: @@ -122,6 +129,7 @@ INSTANTIATE_TEST_CASE_P(flowcell_unit_tests, flowcell_plot_tests, ::testing::ValuesIn(plot_flowcell_generators)); + //--------------------------------------------------------------------------------------------------------------------- // Regression test section //--------------------------------------------------------------------------------------------------------------------- diff --git a/src/tests/interop/logic/plot_heatmap_test.cpp b/src/tests/interop/logic/plot_heatmap_test.cpp index e8e460e10..7cf7b03e3 100644 --- a/src/tests/interop/logic/plot_heatmap_test.cpp +++ b/src/tests/interop/logic/plot_heatmap_test.cpp @@ -13,6 +13,8 @@ #include "interop/model/plot/filter_options.h" #include "src/tests/interop/logic/inc/empty_plot_test_generator.h" #include "src/tests/interop/logic/inc/plot_regression_test_generator.h" +#include "src/tests/interop/metrics/inc/q_metrics_test.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop::model::metrics; @@ -48,6 +50,22 @@ TEST_P(heatmap_plot_tests, plot_data) } } +TEST(heatmap_plot_tests, q_by_lane_metric) +{ + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + + run_metrics metrics(run_info); + q_metric_v6::create_expected(metrics.get()); + metrics.run_info(run_info); + metrics.legacy_channel_update(constants::HiSeq); + metrics.finalize_after_load(); + metrics.get().clear(); + EXPECT_EQ(logic::plot::count_rows_for_heatmap(metrics), 3u); + EXPECT_EQ(logic::plot::count_columns_for_heatmap(metrics), 40u); + +} + //--------------------------------------------------------------------------------------------------------------------- // Unit test parameter section //--------------------------------------------------------------------------------------------------------------------- diff --git a/src/tests/interop/logic/plot_logic_test.cpp b/src/tests/interop/logic/plot_logic_test.cpp index 7b27d3273..5f13a69e4 100644 --- a/src/tests/interop/logic/plot_logic_test.cpp +++ b/src/tests/interop/logic/plot_logic_test.cpp @@ -19,6 +19,7 @@ #include "src/tests/interop/metrics/inc/index_metrics_test.h" #include "src/tests/interop/inc/generic_fixture.h" #include "src/tests/interop/logic/inc/metric_filter_iterator.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop; using namespace illumina::interop::unittest; @@ -39,13 +40,11 @@ TEST(plot_logic, intensity_by_cycle) const size_t channel_count = 4; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(2, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.set_naming_method(constants::FourDigit); - metrics.legacy_channel_update(constants::HiSeq); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::extraction_metric_v2::create_expected(metrics.get()); @@ -55,7 +54,7 @@ TEST(plot_logic, intensity_by_cycle) ASSERT_EQ(data.size(), channel_count); EXPECT_EQ(data.x_axis().label(), "Cycle"); EXPECT_EQ(data.y_axis().label(), "Intensity"); - EXPECT_EQ(data.title(), "All Lanes All Channels"); + EXPECT_EQ(data.title(), "H7MF5BBXX All Lanes All Channels"); EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); EXPECT_NEAR(data.y_axis().min(), 0.0f, tol); EXPECT_NEAR(data.x_axis().max(), 3.0f, tol); @@ -78,13 +77,11 @@ TEST(plot_logic, intensity_by_cycle_empty_interop) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(2, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.set_naming_method(constants::FourDigit); - metrics.legacy_channel_update(constants::HiSeq); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); model::plot::plot_data data; logic::plot::plot_by_cycle(metrics, constants::Intensity, options, data); @@ -104,20 +101,11 @@ TEST(plot_logic, pf_clusters_by_lane) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); - metrics.set_naming_method(constants::FourDigit); - metrics.legacy_channel_update(constants::HiSeq); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::tile_metric_v2::create_expected(metrics.get()); @@ -128,7 +116,7 @@ TEST(plot_logic, pf_clusters_by_lane) EXPECT_NEAR(expected_val, data[0][0].y(), tol); //data.size() refers to the number of series in the collection ASSERT_EQ(data.size(), 1u); - EXPECT_EQ(data.title(), ""); + EXPECT_EQ(data.title(), "H7MF5BBXX"); EXPECT_EQ(data.x_axis().label(), "Lane"); EXPECT_EQ(data.y_axis().label(), "Clusters PF"); EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); @@ -144,13 +132,11 @@ TEST(plot_logic, pf_clusters_by_lane_empty_interop) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.set_naming_method(constants::FourDigit); - metrics.legacy_channel_update(constants::HiSeq); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); model::plot::plot_data data; logic::plot::plot_by_lane(metrics, constants::ClusterCountPF, options, data); @@ -173,13 +159,11 @@ TEST(plot_logic, q_score_histogram) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::q_metric_v6::create_expected(metrics.get()); metrics.finalize_after_load(); @@ -202,7 +186,7 @@ TEST(plot_logic, q_score_histogram) } } - EXPECT_EQ(data.title(), "All Lanes"); + EXPECT_EQ(data.title(), "H7MF5BBXX All Lanes"); EXPECT_EQ(data.x_axis().label(), "Q Score"); EXPECT_EQ(data.y_axis().label(), "Total (million)"); EXPECT_NEAR(data.x_axis().min(), 1.0f, tol); @@ -217,13 +201,11 @@ TEST(plot_logic, q_score_histogram_empty_interop) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); metrics.finalize_after_load(); @@ -247,13 +229,11 @@ TEST(plot_logic, q_score_heatmap) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::q_metric_v6::create_expected(metrics.get()); metrics.finalize_after_load(); @@ -262,7 +242,7 @@ TEST(plot_logic, q_score_heatmap) logic::plot::plot_qscore_heatmap(metrics, options, data); ASSERT_EQ(data.row_count(), 3u); ASSERT_EQ(data.column_count(), 40u); - EXPECT_EQ(data.title(), "All Lanes"); + EXPECT_EQ(data.title(), "H7MF5BBXX All Lanes"); EXPECT_EQ(data.x_axis().label(), "Cycle"); EXPECT_EQ(data.y_axis().label(), "Q Score"); EXPECT_NEAR(data.x_axis().min(), 0.0f, tol); @@ -299,13 +279,11 @@ TEST(plot_logic, q_score_heatmap_empty_interop) const float tol = 1e-3f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), model::run::image_dimensions(), reads)); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); metrics.finalize_after_load(); @@ -327,20 +305,11 @@ TEST(plot_logic, q_score_heatmap_buffer) const float tol = 1e-5f; model::metrics::run_metrics metrics; model::plot::filter_options options(constants::FourDigit); - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(8, 2, 2, 16), - std::vector(), - model::run::image_dimensions(), - reads - )); - metrics.legacy_channel_update(constants::HiSeq); - metrics.set_naming_method(constants::FourDigit); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::q_metric_v6::create_expected(metrics.get()); metrics.finalize_after_load(); @@ -371,20 +340,13 @@ TEST(plot_logic, flowcell_map) const float tol = 1e-3f; const size_t num_lanes = 8; model::metrics::run_metrics metrics; - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(num_lanes, 2, 2, 36, 1, 1, std::vector(), constants::FourDigit), - std::vector(), - model::run::image_dimensions(), - reads - )); + + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); + model::plot::filter_options options(constants::FourDigit, ALL_IDS, 0, constants::A, ALL_IDS, 1, 1); - metrics.legacy_channel_update(constants::HiSeq); unittest::extraction_metric_v2::create_expected(metrics.get()); ASSERT_GT(metrics.get< model::metrics::extraction_metric >().size(), 0u); @@ -398,8 +360,8 @@ TEST(plot_logic, flowcell_map) actual_values.push_back(std::numeric_limits::quiet_NaN()); } float expected_vals[] = {302, 312, 349}; - std::pair indices[] = {std::make_pair(13, 6), std::make_pair(49, 6), - std::make_pair(85, 6)}; + std::pair indices[] = {std::make_pair(13, 6), std::make_pair(41, 6), + std::make_pair(69, 6)}; size_t k = 0; for (size_t i = 0; i < data.row_count(); i++) { @@ -419,14 +381,14 @@ TEST(plot_logic, flowcell_map) { if (!std::isnan(actual_values[val])) { - ASSERT_EQ(val / data.column_count(), indices[m].second); - ASSERT_EQ(val % data.column_count(), indices[m].first); + EXPECT_EQ(val / data.column_count(), indices[m].second); + EXPECT_EQ(val % data.column_count(), indices[m].first); m++; } } ASSERT_EQ(data.row_count(), num_lanes); - EXPECT_EQ(data.title(), "Intensity"); + EXPECT_EQ(data.title(), "H7MF5BBXX Intensity"); EXPECT_EQ(data.saxis().label(), "Intensity"); EXPECT_NEAR(data.saxis().min(), 302.0f, tol); EXPECT_NEAR(data.saxis().max(), 349.0f, tol); @@ -437,22 +399,14 @@ TEST(plot_logic, flowcell_map_empty_interop) { const model::plot::filter_options::id_t ALL_IDS = model::plot::filter_options::ALL_IDS; const float tol = 1e-3f; - const size_t num_lanes = 8; - model::metrics::run_metrics metrics; - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info(model::run::info( - "", - "", - 1, - model::run::flowcell_layout(num_lanes, 2, 2, 36, 1, 1, std::vector(), constants::FourDigit), - std::vector(), - model::run::image_dimensions(), - reads - )); + model::run::info run_info; + const model::run::read_info read_array[]={ + model::run::read_info(1, 1, 26), + model::run::read_info(2, 27, 76) + }; + hiseq4k_run_info::create_expected(run_info, util::to_vector(read_array)); + model::metrics::run_metrics metrics(run_info); model::plot::filter_options options(constants::FourDigit, ALL_IDS, 0, constants::A, ALL_IDS, 1, 1); - metrics.legacy_channel_update(constants::HiSeq); model::plot::flowcell_data data; logic::plot::plot_flowcell_map(metrics, constants::Intensity, options, data); @@ -469,14 +423,10 @@ TEST(plot_logic, sample_qc) { const float tol = 1e-3f; model::metrics::run_metrics metrics; - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info( - model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 36, 1, 1, std::vector(), - constants::FourDigit), - std::vector(), model::run::image_dimensions(), reads)); - metrics.legacy_channel_update(constants::HiSeq); + + model::run::info run_info; + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); unittest::index_metric_v1::create_expected(metrics.get()); unittest::tile_metric_v2::create_expected(metrics.get()); @@ -508,15 +458,12 @@ TEST(plot_logic, sample_qc_empty_interop) { const float tol = 1e-3f; model::metrics::run_metrics metrics; - std::vector reads(2); - reads[0] = model::run::read_info(1, 1, 26); - reads[1] = model::run::read_info(2, 27, 76); - metrics.run_info( - model::run::info("", "", 1, model::run::flowcell_layout(8, 2, 2, 36, 1, 1, std::vector(), - constants::FourDigit), - std::vector(), model::run::image_dimensions(), reads)); - metrics.legacy_channel_update(constants::HiSeq); + + model::run::info run_info; + + hiseq4k_run_info::create_expected(run_info); + metrics.run_info(run_info); model::plot::plot_data data; logic::plot::plot_sample_qc(metrics, 1, data); ASSERT_EQ(data.size(), 0u); diff --git a/src/tests/interop/logic/summary_metrics_test.cpp b/src/tests/interop/logic/summary_metrics_test.cpp index 51cdde791..31d642332 100644 --- a/src/tests/interop/logic/summary_metrics_test.cpp +++ b/src/tests/interop/logic/summary_metrics_test.cpp @@ -18,6 +18,7 @@ #include "src/tests/interop/metrics/inc/tile_metrics_test.h" #include "src/tests/interop/metrics/inc/q_metrics_test.h" #include "src/tests/interop/inc/abstract_regression_test_generator.h" +#include "src/tests/interop/run/info_test.h" #include "src/tests/interop/metrics/inc/phasing_metrics_test.h" using namespace illumina::interop::model::summary; @@ -369,30 +370,10 @@ TEST_P(run_summary_tests, surface_summary) TEST(summary_metrics_test, cycle_35_cycle_34_tile) { - const size_t lane_count = 8; - const size_t surface_count = 2; - const size_t swath_count = 4; - const size_t tile_count = 99; - const size_t sections_per_lane = 1; - const size_t lanes_per_section = 1; - std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); - std::vector reads; - reads.push_back(model::run::read_info(1, 1, 36)); - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count, - surface_count, - swath_count, - tile_count, - sections_per_lane, - lanes_per_section), - channels, - model::run::image_dimensions(), - reads); - run_info.set_naming_method(constants::FourDigit); + + model::run::info run_info; + model::run::read_info reads[] = {model::run::read_info(1, 1, 36)}; + hiseq4k_run_info::create_expected(run_info, util::to_vector(reads)); model::metrics::run_metrics expected_run_metrics(run_info); model::metric_base::metric_set &expected_error_metrics = @@ -435,32 +416,15 @@ TEST(summary_metrics_test, cycle_35_cycle_34_tile) TEST(summary_metrics_test, clear_run_metrics) // TODO Expand to catch everything: probably use a fixture and the methods above { const float tol = 1e-9f; - const size_t lane_count = 8; - const size_t surface_count = 2; - const size_t swath_count = 4; - const size_t tile_count = 99; - const size_t sections_per_lane = 1; - const size_t lanes_per_section = 1; - std::vector channels; - channels.push_back("Red"); - channels.push_back("Green"); - std::vector reads; - reads.push_back(model::run::read_info(1, 1, 36)); - reads.push_back(model::run::read_info(2, 37, 42)); - reads.push_back(model::run::read_info(3, 43, 80)); - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count, - surface_count, - swath_count, - tile_count, - sections_per_lane, - lanes_per_section), - channels, - model::run::image_dimensions(), - reads); - run_info.set_naming_method(constants::FourDigit); + + + model::run::info run_info; + const model::run::read_info read_array[]={ + model::run::read_info(1, 1, 36), + model::run::read_info(2, 37, 42), + model::run::read_info(3, 43, 80) + }; + hiseq4k_run_info::create_expected(run_info, util::to_vector(read_array)); model::metrics::run_metrics full_metrics(run_info); tile_metric_v2::create_expected(full_metrics.get(), run_info); @@ -553,18 +517,14 @@ class run_summary_generator : public abstract_generator(), run_info); diff --git a/src/tests/interop/metrics/inc/error_metrics_test.h b/src/tests/interop/metrics/inc/error_metrics_test.h index 08e9a356d..ac35a36ec 100644 --- a/src/tests/interop/metrics/inc/error_metrics_test.h +++ b/src/tests/interop/metrics/inc/error_metrics_test.h @@ -97,7 +97,8 @@ namespace illumina { namespace interop { namespace unittest * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these - * statistics. + * statistics. This is calculated only over "usable cycles". If a read has less than 35 cycles, then the last cycle + * is dropped. * * @section error_metrics_requirement_50_lane % Error After 50 Cycles Per Lane * @@ -106,7 +107,8 @@ namespace illumina { namespace interop { namespace unittest * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these - * statistics. + * statistics. This is calculated only over "usable cycles". If a read has less than 50 cycles, then the last cycle + * is dropped. * * @section error_metrics_requirement_75_lane % Error After 75 Cycles Per Lane * @@ -115,7 +117,8 @@ namespace illumina { namespace interop { namespace unittest * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these - * statistics. + * statistics. This is calculated only over "usable cycles". If a read has less than 75 cycles, then the last cycle + * is dropped. * * @section error_metrics_requirement_100_lane % Error After 100 Cycles Per Lane * @@ -124,7 +127,8 @@ namespace illumina { namespace interop { namespace unittest * a given tile, it is excluded from the averaging. If no tiles have any error statistics for a given read, a * value of 0 is returned. Note that the base unit for this is the error rate for a single tile - in other words, * we average the per-cycle error rate across all the relevant cycles for a tile before calculating these - * statistics. + * statistics. This is calculated only over "usable cycles". If a read has less than 100 cycles, then the last cycle + * is dropped. * * @section error_metrics_requirement_error % Error * @@ -134,7 +138,8 @@ namespace illumina { namespace interop { namespace unittest * an error rate value is not present for any cycles in the read for a given tile, it is excluded from the * averaging. If no tiles have any error statistics for a given read, a value of 0 is returned. Note that the * base unit for this is the error rate for a single tile - in other words, we average the per-cycle error - * rate across all the relevant cycles for a tile before calculating these statistics + * rate across all the relevant cycles for a tile before calculating these statistics. This is calculated only over + * "usable cycles", so the last cycle is not included. * * @subsection error_metrics_requirement_error_read Read * @@ -142,7 +147,8 @@ namespace illumina { namespace interop { namespace unittest * not present for any cycles in the read for a given tile, it is excluded from the averaging. If no tiles have * any error statistics for a given read, a value of 0 is returned. Note that the base unit for this is the error * rate for a single tile - in other words, we average the per-cycle error rate across all the relevant cycles for - * a tile before calculating these statistics + * a tile before calculating these statistics. This is calculated only over "usable cycles", so the last cycle is + * not included. * * @subsection error_metrics_requirement_error_non_index Non-Indexed Total * @@ -150,7 +156,8 @@ namespace illumina { namespace interop { namespace unittest * present for any cycles in the read for a given tile, it is excluded from the averaging. If no tiles have any * error statistics for any non-indexed read, a value of 0 is returned. Note that the base unit for this is the * error rate for a single tile - in other words, we average the per-cycle error rate across all the relevant - * cycles for a tile before calculating these statistics + * cycles for a tile before calculating these statistics. This is calculated only over "usable cycles", so the + * last cycle is not included. * * @subsection error_metrics_requirement_error_total Total * @@ -158,7 +165,8 @@ namespace illumina { namespace interop { namespace unittest * any cycles in the read for a given tile, it is excluded from the averaging. If no tiles have any error * statistics for any read, a value of 0 is returned. Note that the base unit for this is the error rate for * a single tile - in other words, we average the per-cycle error rate across all the relevant cycles for a - * tile before calculating these statistics + * tile before calculating these statistics. This is calculated only over "usable cycles", so the + * last cycle is not included. * */ diff --git a/src/tests/interop/metrics/metric_regression_tests.cpp b/src/tests/interop/metrics/metric_regression_tests.cpp new file mode 100644 index 000000000..3db2c955d --- /dev/null +++ b/src/tests/interop/metrics/metric_regression_tests.cpp @@ -0,0 +1,124 @@ +/** Regression tests for the run metrics + * + * + * @file + * @date 2/24/17 + * @version 1.0 + * @copyright GNU Public License. + */ +#include +#include "interop/model/run_metrics.h" +#include "interop/logic/utils/enums.h" +#include "src/tests/interop/inc/abstract_regression_test_generator.h" + +using namespace illumina::interop; +using namespace illumina::interop::unittest; + + +/** Setup for tests that use a single, actual run_metrics object */ +struct run_metrics_tests : public generic_test_fixture {}; + + +/** Test if we can compute buffer size for every non-empty metric + */ +TEST_P(run_metrics_tests, compute_buffer_size) +{ + if (skip_test) return; + ASSERT_TRUE(fixture_test_result); + + for(size_t i=0;i(constants::MetricCount);++i) + { + const constants::metric_group group = static_cast(i); + if(group == constants::DynamicPhasing) continue; // This does not have a format + if(actual.is_group_empty(group)) continue; + EXPECT_NO_THROW(actual.calculate_buffer_size(group)) << constants::to_string(group); + } +} + +/** Read in the actual run metrics, leave expected empty + */ +class regression_test_metric_generator : public abstract_regression_test_generator +{ + typedef abstract_regression_test_generator parent_t; +public: + /** Constructor + * + * @param test_dir sub folder where tests are stored + */ + regression_test_metric_generator(const std::string &test_dir) : parent_t(test_dir) + {} + + /** Constructor + * + * @param run_folder run folder with data + * @param test_dir sub folder where tests are stored + */ + regression_test_metric_generator(const std::string &run_folder, const std::string &test_dir) : parent_t(run_folder, + test_dir) + {} + +protected: + /** This function does nothing + * + * @return true + */ + bool read_expected(const std::string &, model::metrics::run_metrics &) const + { + return true; + } + + /** Read the actual data from the run folder + * + * @param run_folder run folder + * @param actual actual model data + * @return true + */ + bool generate_actual(const std::string &run_folder, model::metrics::run_metrics &actual) const + { + actual.read(run_folder); + return true; + } + + /** This function does nothing + * + * @param baseline_file baseline file + * @param actual actual model data + */ + bool write_actual(const std::string &, const model::metrics::run_metrics &) const + { + return true; + } + /** Create a copy of the current object with the given run folder + * + * @param run_folder run folder + * @return pointer to new copy + */ + base_t clone(const std::string& run_folder)const + { + return new regression_test_metric_generator(run_folder, m_test_dir); + } + + /** Create a copy of the current object + * + * @return pointer to new copy + */ + base_t clone() const + { + return new regression_test_metric_generator(*this); + } + + /** Write generator info to output stream + * + * @param out output stream + */ + void write(std::ostream &out) const + { + out << "regression_test_metric_generator - " << io::basename(m_run_folder); + } +}; + +regression_test_metric_generator run_metrics_regression_gen("empty"); + +INSTANTIATE_TEST_CASE_P(run_metrics_regression_test, + run_metrics_tests, + ProxyValuesIn(run_metrics_regression_gen, regression_test_data::instance().files())); diff --git a/src/tests/interop/metrics/metric_streams_test.cpp b/src/tests/interop/metrics/metric_streams_test.cpp index b64754c84..b61232ec0 100644 --- a/src/tests/interop/metrics/metric_streams_test.cpp +++ b/src/tests/interop/metrics/metric_streams_test.cpp @@ -26,6 +26,8 @@ struct metric_stream_test : public ::testing::Test, public TestSetup { /** Type of metric set */ typedef typename TestSetup::metric_set_t metric_set_t; + /** Type of metric */ + typedef typename metric_set_t::metric_type metric_t; /** Constructor */ metric_stream_test() { @@ -72,15 +74,27 @@ TYPED_TEST_P(metric_stream_test, test_header_size) */ TYPED_TEST_P(metric_stream_test, test_read_data_size) { - if (TypeParam::disable_binary_data_size || TypeParam::disable_binary_data) return; + typedef typename TestFixture::metric_t metric_t; std::string tmp = std::string(TestFixture::expected); typename TypeParam::metric_set_t metrics; io::read_interop_from_string(tmp, metrics); - if(io::is_multi_record(metrics)) return; + if(static_cast(metric_t::TYPE) == constants::Tile && TypeParam::VERSION == 2) + return; // This contrived exampled is not supported, it includes things like control metrics, which are ignored const size_t expected_size = io::compute_buffer_size(metrics); EXPECT_EQ(tmp.size(), expected_size); } +/** Confirm the header size matches what is read + */ +TYPED_TEST_P(metric_stream_test, test_write_data_size) +{ + typename TypeParam::metric_set_t metrics; + TypeParam::create_expected(metrics); + const size_t expected_size = io::compute_buffer_size(metrics); + std::vector< ::uint8_t > buffer(expected_size); + EXPECT_NO_THROW(io::write_interop_to_buffer(metrics, &buffer.front(), buffer.size())); +} + TEST(metric_stream_test, list_filenames) { std::vector error_metric_files; @@ -97,7 +111,8 @@ TEST(metric_stream_test, list_filenames) REGISTER_TYPED_TEST_CASE_P(metric_stream_test, test_read_data_size, test_header_size, - test_write_read_binary_data + test_write_read_binary_data, + test_write_data_size ); diff --git a/src/tests/interop/metrics/q_by_lane_metric_test.cpp b/src/tests/interop/metrics/q_by_lane_metric_test.cpp index 45e514786..592007535 100644 --- a/src/tests/interop/metrics/q_by_lane_metric_test.cpp +++ b/src/tests/interop/metrics/q_by_lane_metric_test.cpp @@ -14,6 +14,7 @@ #include "src/tests/interop/inc/proxy_parameter_generator.h" #include "src/tests/interop/metrics/inc/metric_generator.h" #include "src/tests/interop/metrics/inc/q_metrics_test.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop; using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; @@ -84,36 +85,14 @@ TEST(run_metrics_q_by_lane_test, test_is_group_empty) // Test that unsupported feature throws an exception TEST(run_metrics_q_by_lane_test, try_write_binned_as_unbinned) { - typedef model::run::flowcell_layout::uint_t uint_t; run_metrics metrics; - - - const uint_t swath_count = 4; - const uint_t tile_count = 99; - const uint_t sections_per_lane = 1; - const uint_t lanes_per_section = 1; - const uint_t lane_count = 8;//expected.max_lane(); - const uint_t surface_count = 2; + model::run::info run_info; const model::run::read_info read_array[]={ model::run::read_info(1, 1, 4, false) }; - std::vector channels; - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count, - surface_count, - swath_count, - tile_count, - sections_per_lane, - lanes_per_section), - channels, - model::run::image_dimensions(), - util::to_vector(read_array)); - + hiseq4k_run_info::create_expected(run_info, util::to_vector(read_array)); q_metric_v6::create_expected(metrics.get()); metrics.run_info(run_info); - metrics.legacy_channel_update(constants::HiSeq); metrics.finalize_after_load(); ASSERT_GT(metrics.get().size(), 0u); diff --git a/src/tests/interop/metrics/q_collapsed_metrics_test.cpp b/src/tests/interop/metrics/q_collapsed_metrics_test.cpp index 81b3a264d..a29c98af9 100644 --- a/src/tests/interop/metrics/q_collapsed_metrics_test.cpp +++ b/src/tests/interop/metrics/q_collapsed_metrics_test.cpp @@ -15,6 +15,7 @@ #include "src/tests/interop/metrics/inc/metric_generator.h" #include "interop/logic/metric/q_metric.h" #include "src/tests/interop/metrics/inc/q_metrics_test.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop::model::metrics; using namespace illumina::interop::model::metric_base; @@ -123,36 +124,15 @@ TEST(q_collapsed_metrics_test, test_convert_write_read) TEST(q_collapsed_metrics_test, compute_buffer_size) { - typedef model::run::flowcell_layout::uint_t uint_t; run_metrics metrics; - - - const uint_t swath_count = 4; - const uint_t tile_count = 99; - const uint_t sections_per_lane = 1; - const uint_t lanes_per_section = 1; - const uint_t lane_count = 8;//expected.max_lane(); - const uint_t surface_count = 2; + model::run::info run_info; const model::run::read_info read_array[]={ model::run::read_info(1, 1, 4, false) }; - std::vector channels; - model::run::info run_info("XX", - "", - 1, - model::run::flowcell_layout(lane_count, - surface_count, - swath_count, - tile_count, - sections_per_lane, - lanes_per_section), - channels, - model::run::image_dimensions(), - util::to_vector(read_array)); + hiseq4k_run_info::create_expected(run_info, util::to_vector(read_array)); q_metric_v6::create_expected(metrics.get()); metrics.run_info(run_info); - metrics.legacy_channel_update(constants::HiSeq); metrics.finalize_after_load(); EXPECT_GT(metrics.get().size(), 0u); diff --git a/src/tests/interop/metrics/q_metrics_test.cpp b/src/tests/interop/metrics/q_metrics_test.cpp index e5b1035d4..b3431f76f 100644 --- a/src/tests/interop/metrics/q_metrics_test.cpp +++ b/src/tests/interop/metrics/q_metrics_test.cpp @@ -136,6 +136,41 @@ TEST(q_metrics_test, test_cumulative) EXPECT_EQ(q_metric_set.get_metric(7, 1114, 3).sum_qscore_cumulative(), qsum); } +TEST(q_metrics_test, test_percent_over_q30_unbinned) +{ + q_score_header header; + q_metric::uint32_vector data(50); + data[28] = 10; + data[29] = 5; // Q30 + data[30] = 5; + q_metric metric(1, 1, 1, data); + + EXPECT_EQ(metric.percent_over_qscore(header.index_for_q_value(30)), 50); +} + +TEST(q_metrics_test, test_percent_over_q30_binned) +{ + q_score_bin rbins[] = { + q_score_bin(0, 9, 6), + q_score_bin(10, 19, 15), + q_score_bin(20, 24, 22), + q_score_bin(25, 29, 27), + q_score_bin(30, 34, 33), + q_score_bin(35, 39, 37), + q_score_bin(40, 49, 40) + }; + q_metric::qscore_bin_vector_type bins(util::to_vector(rbins)); + q_score_header header(bins); + + q_metric::uint32_vector data(bins.size()); + data[3] = 10; + data[4] = 5; + data[5] = 5; + q_metric metric(1, 1, 1, data); + + EXPECT_EQ(metric.percent_over_qscore(header.index_for_q_value(30)), 50); +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Setup regression test //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/tests/interop/metrics/run_metric_test.cpp b/src/tests/interop/metrics/run_metric_test.cpp index 2518ccc7b..a24afeb9e 100644 --- a/src/tests/interop/metrics/run_metric_test.cpp +++ b/src/tests/interop/metrics/run_metric_test.cpp @@ -39,9 +39,10 @@ TYPED_TEST_P(run_metric_test, on_demand_not_clear) { typedef typename TestFixture::metric_set_t metric_set_t; metric_set_t& metric_set = TestFixture::expected.template get(); + const size_t thread_count = 1; try{ std::vector valid_to_load; - TestFixture::expected.read("/File/not/found.aaa", valid_to_load); + TestFixture::expected.read("/File/not/found.aaa", valid_to_load, thread_count); }catch(const xml::xml_file_not_found_exception&){} EXPECT_FALSE(metric_set.empty()); } diff --git a/src/tests/interop/metrics/tile_metrics_test.cpp b/src/tests/interop/metrics/tile_metrics_test.cpp index f79bc24f7..fd5781a30 100644 --- a/src/tests/interop/metrics/tile_metrics_test.cpp +++ b/src/tests/interop/metrics/tile_metrics_test.cpp @@ -51,10 +51,7 @@ class write_metric_generator : public abstract_generator< typename Gen::metric_s std::ostringstream fout; illumina::interop::io::write_metrics(fout, expected); actual.clear(); - - try{ - io::read_interop_from_string(fout.str(), actual); - }catch(const io::incomplete_file_exception&){} + io::read_interop_from_string(fout.str(), actual); return ::testing::AssertionSuccess(); } /** Create a copy of this object diff --git a/src/tests/interop/run/info_test.cpp b/src/tests/interop/run/info_test.cpp index 536c353c6..3379ab497 100644 --- a/src/tests/interop/run/info_test.cpp +++ b/src/tests/interop/run/info_test.cpp @@ -9,265 +9,105 @@ #include #include #include +#include "interop/util/length_of.h" #include "interop/constants/enums.h" #include "interop/model/run/info.h" +#include "src/tests/interop/run/info_test.h" using namespace illumina::interop::model; using namespace illumina::interop; +using namespace illumina::interop::unittest; -/** Convert an array to a vector - * - * Determines the length of the stack array automatically. - * - * @param vals array pointer - * @return vector of values - */ -template -std::vector to_vector(const T (&vals)[N]) -{ - return std::vector(vals, vals + N); -} -static void test_helper_run_info_parse_xml(const run::info& expected_run_info, - std::string& xml_file) +/** Fixture to list enum values */ +template +struct run_info_fixture : public ::testing::Test, public T { - run::info run_info; - run_info.parse(&xml_file[0]); - - EXPECT_EQ(run_info.name(), expected_run_info.name()); - EXPECT_EQ(run_info.date(), expected_run_info.date()); - EXPECT_EQ(run_info.version(), expected_run_info.version()); - EXPECT_EQ(run_info.flowcell().lane_count(), expected_run_info.flowcell().lane_count()); - EXPECT_EQ(run_info.flowcell().surface_count(), expected_run_info.flowcell().surface_count()); - EXPECT_EQ(run_info.flowcell().swath_count(), expected_run_info.flowcell().swath_count()); - EXPECT_EQ(run_info.flowcell().tile_count(), expected_run_info.flowcell().tile_count()); - EXPECT_EQ(run_info.flowcell().sections_per_lane(), expected_run_info.flowcell().sections_per_lane()); - EXPECT_EQ(run_info.flowcell().lanes_per_section(), expected_run_info.flowcell().lanes_per_section()); - EXPECT_EQ(run_info.flowcell().naming_method(), expected_run_info.flowcell().naming_method()); - EXPECT_EQ(run_info.dimensions_of_image().width(), expected_run_info.dimensions_of_image().width()); - EXPECT_EQ(run_info.dimensions_of_image().height(), expected_run_info.dimensions_of_image().height()); - EXPECT_EQ(run_info.channels().size(), expected_run_info.channels().size()); - - for(size_t i=0;i +struct read_fixture +{ + /** Constructor */ + read_fixture() { - EXPECT_EQ(run_info.flowcell().tiles()[i], expected_run_info.flowcell().tiles()[i]); + T::create_expected(expected_run_info); + std::string xml_file; + T::create_string(xml_file); + run_info.parse(&xml_file[0]); } -} -/**120705_11618Unbin1R2I - * @class illumina::interop::model::run::info - * @test Confirm MiSeq version of the run info XML format is properly parsed - * - */ -TEST(RunInfo, ParseXML_MiSeq) -{ - std::string xml_file = "\n" - "\n" - " \n" - " 000000000-A12V4\n" - " M00903\n" - " 120705\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - ""; - - //std::string Tiles[] = {}; - const std::vector Tiles; - //std::string ImageChannels[] = {}; - const std::vector ImageChannels; - const run::read_info Reads[] = { - /*Number, IsIndexedRead, CycleStart, CycleEnd*/ - run::read_info(1, 1, 251, false), - run::read_info(2, 252, 263, true), - run::read_info(3, 264, 514, false) - }; - const constants::tile_naming_method TileNamingConvention= constants::UnknownTileNamingMethod; - const size_t LaneCount = 1; - const size_t SurfaceCount = 2; - const size_t SwathCount = 1; - const size_t TileCount=14; - const size_t SectionPerLane = 1; - const size_t LanePerSection = 1; - const run::flowcell_layout FlowcellLayout(LaneCount, - SurfaceCount, - SwathCount, - TileCount, - SectionPerLane, - LanePerSection, - Tiles, - TileNamingConvention); - - const std::string Run_Id = "120705_M00903_0009_A000000000-A12V4"; - const std::string Date = "120705"; - const run::info::uint_t version = 2; - const run::image_dimensions ImageDimensions(/*Width*/ 0, /*Height*/ 0); - const run::info expected_run_info(Run_Id, - Date , - version, - FlowcellLayout, - ImageChannels, - ImageDimensions, - to_vector(Reads)); - - test_helper_run_info_parse_xml(expected_run_info, xml_file); -} - -/**131212_221Bin1R0I - * @class illumina::interop::model::run::info - * @test Confirm HiSeq version of the run info XML format is properly parsed - * - */ -TEST(RunInfo, ParseXML_HiSeq) + run::info expected_run_info; + run::info run_info; +}; +template +struct write_read_fixture { - std::string xml_file = "\n" - "\n" - " \n" - " 02D224DRR\n" - " 1270\n" - " 131212\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " 1_1203\n" - "\t\n" - " \n" - " \n" - " \n" - " \n" - " a\n" - " c\n" - " g\n" - " t\n" - " \n" - " \n" - " 1\n" - " 2\n" - " 3\n" - " 4\n" - " 5\n" - " 6\n" - " 7\n" - " 8\n" - " \n" - " \n" - ""; - - - - const std::string Tiles[] = {"1_1203"}; - const std::string ImageChannels[] = {"a", "c", "g", "t"}; - const run::read_info Reads[] = { - /*Number, IsIndexedRead, CycleStart, CycleEnd*/ - run::read_info(1, 1, 30, false) - }; - const constants::tile_naming_method TileNamingConvention= constants::FourDigit; - const size_t LaneCount = 8; - const size_t SurfaceCount = 2; - const size_t SwathCount = 2; - const size_t TileCount=24; - const size_t SectionPerLane = 1; - const size_t LanePerSection = 1; - const run::flowcell_layout FlowcellLayout(LaneCount, - SurfaceCount, - SwathCount, - TileCount, - SectionPerLane, - LanePerSection, - to_vector(Tiles), - TileNamingConvention); - - const std::string Run_Id = "131212_1270_0870_A02D224"; - const std::string Date = "131212"; - const run::info::uint_t version = 2; - const run::image_dimensions ImageDimensions(/*Width*/ 3188, /*Height*/ 7244); - const run::info expected_run_info(Run_Id, - Date , - version, - FlowcellLayout, - to_vector(ImageChannels), - ImageDimensions, - to_vector(Reads)); - - test_helper_run_info_parse_xml(expected_run_info, xml_file); -} - -/**6471489_213Bin1R0I - * @class illumina::interop::model::run::info - * @test Confirm NexSeq version of the run info XML format is properly parsed - * - */ -TEST(RunInfo, ParseXML_NextSeq) + /** Constructor */ + write_read_fixture() + { + T::create_expected(expected_run_info); + std::ostringstream fout; + expected_run_info.write(fout); + std::string tmp = fout.str(); + run_info.parse(&tmp[0]); + } + run::info expected_run_info; + run::info run_info; +}; +typedef ::testing::Types< + read_fixture, + read_fixture, + read_fixture, + read_fixture, + read_fixture, + read_fixture, + + + write_read_fixture, + write_read_fixture, + write_read_fixture, + write_read_fixture, + write_read_fixture, + write_read_fixture +> run_info_list; +TYPED_TEST_CASE(run_info_fixture, run_info_list); + +/** Confirm that every enum does not conflict with unknown */ +TYPED_TEST(run_info_fixture, run_info_test) { - std::string xml_file = "\n" - "\n" - " \n" - " XXXXXXXXX\n" - " NS500128\n" - " 140502\n" - " \n" - " \n" - " \n" - " \n" - " \n" - ""; - - - - //std::string Tiles[] = {}; - const std::vector Tiles; - //std::string ImageChannels[] = {}; - const std::vector ImageChannels; - const run::read_info Reads[] = { - /*Number, IsIndexedRead, CycleStart, CycleEnd*/ - run::read_info(1, 1, 75, false) - }; - const constants::tile_naming_method TileNamingConvention= constants::UnknownTileNamingMethod; - const size_t LaneCount = 4; - const size_t SurfaceCount = 2; - const size_t SwathCount = 3; - const size_t TileCount=12; - const size_t SectionPerLane = 3; - const size_t LanePerSection = 2; - const run::flowcell_layout FlowcellLayout(LaneCount, - SurfaceCount, - SwathCount, - TileCount, - SectionPerLane, - LanePerSection, - Tiles, - TileNamingConvention); - - const std::string Run_Id = "140502_NS500128_0072_AH0MNBAGXX_TsCancer_1.6pM"; - const std::string Date = "140502"; - const run::info::uint_t version = 2; - const run::image_dimensions ImageDimensions(/*Width*/ 0, /*Height*/ 0); - const run::info expected_run_info(Run_Id, - Date , - version, - FlowcellLayout, - ImageChannels, - ImageDimensions, - to_vector(Reads)); - - test_helper_run_info_parse_xml(expected_run_info, xml_file); + EXPECT_EQ(TestFixture::run_info.name(), TestFixture::expected_run_info.name()); + EXPECT_EQ(TestFixture::run_info.date(), TestFixture::expected_run_info.date()); + EXPECT_EQ(TestFixture::run_info.instrument_name(), TestFixture::expected_run_info.instrument_name()); + EXPECT_EQ(TestFixture::run_info.flowcell_id(), TestFixture::expected_run_info.flowcell_id()); + EXPECT_EQ(TestFixture::run_info.run_number(), TestFixture::expected_run_info.run_number()); + EXPECT_EQ(TestFixture::run_info.version(), TestFixture::expected_run_info.version()); + EXPECT_EQ(TestFixture::run_info.flowcell().lane_count(), TestFixture::expected_run_info.flowcell().lane_count()); + EXPECT_EQ(TestFixture::run_info.flowcell().surface_count(), TestFixture::expected_run_info.flowcell().surface_count()); + EXPECT_EQ(TestFixture::run_info.flowcell().swath_count(), TestFixture::expected_run_info.flowcell().swath_count()); + EXPECT_EQ(TestFixture::run_info.flowcell().tile_count(), TestFixture::expected_run_info.flowcell().tile_count()); + EXPECT_EQ(TestFixture::run_info.flowcell().sections_per_lane(), TestFixture::expected_run_info.flowcell().sections_per_lane()); + EXPECT_EQ(TestFixture::run_info.flowcell().lanes_per_section(), TestFixture::expected_run_info.flowcell().lanes_per_section()); + EXPECT_EQ(TestFixture::run_info.flowcell().naming_method(), TestFixture::expected_run_info.flowcell().naming_method()); + EXPECT_EQ(TestFixture::run_info.dimensions_of_image().width(), TestFixture::expected_run_info.dimensions_of_image().width()); + EXPECT_EQ(TestFixture::run_info.dimensions_of_image().height(), TestFixture::expected_run_info.dimensions_of_image().height()); + EXPECT_EQ(TestFixture::run_info.channels().size(), TestFixture::expected_run_info.channels().size()); + + for(size_t i=0;i\n" + "\n" + " \n" + " 000000000-A12V4\n" + " M00903\n" + " 120705\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + ""; + } + /** Generate a standard MiSeq runinfo with an arbitrary read structure + * + * @param expected_run_info + * @param reads + */ + static void create_expected(model::run::info& expected_run_info, const std::vector& reads) + { + expected_run_info = model::run::info("120705_M00903_0009_A000000000-A12V4" /* run id */, + "120705" /* date */ , + "M00903" /* instrument name*/, + 8 /* run number */, + 2 /** RunInfo version */, + model::run::flowcell_layout(1 /*lane count*/, + 2 /*surface count*/, + 1 /*swath count */, + 14 /*tile count */, + 1 /* sections per lane */, + 1 /* lanes per section*/, + std::vector () /* Tiles */, + constants::UnknownTileNamingMethod, + "000000000-A12V4" /* flowcell id */), + std::vector() /*Image channels */, + model::run::image_dimensions(/*Width*/ 0, /*Height*/ 0), + reads); + } + /** Generate expected run info relating to file representation + * + * @param expected_run_info destination run info + */ + static void create_expected(model::run::info& expected_run_info) + { + const model::run::read_info reads[] = { + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + model::run::read_info(1, 1, 251, false), + model::run::read_info(2, 252, 263, true), + model::run::read_info(3, 264, 514, false) + }; + create_expected(expected_run_info, util::to_vector(reads)); + } + }; + + + /** Provide an example HiSeq 2500 RunInfo v2 + */ + struct hiseq2500_run_info + { + /** Generate a file representation of the RunInfo in a string + * + * @param data destination string + */ + static void create_string(std::string& data) + { + data= "\n" + "\n" + " \n" + " HCTK2BCXX\n" + " D00154\n" + " 151014\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " 1\n" + " 2\n" + " \n" + " \n" + ""; + } + /** Generate a standard runinfo with an arbitrary read structure + * + * @param expected_run_info destination run info + * @param reads arbitrary read structure + */ + static void create_expected(model::run::info& expected_run_info, const std::vector& reads) + { + expected_run_info = model::run::info("151014_D00154_0539_AHCTK2BCXX_HiSeq 2500_TruSeq Exome_12 replicates of NA12878" /* run id */, + "151014" /* date */ , + "D00154" /* instrument name*/, + 539 /* run number */, + 2 /** RunInfo version */, + model::run::flowcell_layout(2 /*lane count*/, + 2 /*surface count*/, + 2 /*swath count */, + 16 /*tile count */, + 1 /* sections per lane */, + 1 /* lanes per section*/, + std::vector() /* Tiles */, + constants::UnknownTileNamingMethod, + "HCTK2BCXX" /* flowcell id */), + std::vector() /*Image channels */, + model::run::image_dimensions(/*Width*/ 0, /*Height*/ 0), + reads); + } + /** Generate expected run info relating to file representation + * + * @param expected_run_info destination run info + */ + static void create_expected(model::run::info& expected_run_info) + { + const model::run::read_info reads[] = { + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + model::run::read_info(1, 1, 76, false), + model::run::read_info(2, 77, 84, true), + model::run::read_info(3, 85, 160, false) + }; + create_expected(expected_run_info, util::to_vector(reads)); + } + }; + + /** Provide an example HiSeq 4k RunInfo v3 + */ + struct hiseq4k_run_info + { + /** Generate a file representation of the RunInfo in a string + * + * @param data destination string + */ + static void create_string(std::string& data) + { + data= "\n" + "\n" + " \n" + " H7MF5BBXX\n" + " 196\n" + " 160223\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " 1_1101\n" + " 1_1102\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " A\n" + " G\n" + " T\n" + " C\n" + " \n" + " \n" + ""; + } + /** Generate a standard runinfo with an arbitrary read structure + * + * @param expected_run_info destination run info + * @param reads arbitrary read structure + */ + static void create_expected(model::run::info& expected_run_info, const std::vector& reads) + { + std::string tiles[] = { + "1_1101", + "1_1102" + }; + const std::string channels[] = {"a", "g", "t", "c"}; + expected_run_info = model::run::info("160223_196_2371_AH7MF5BBXX" /* run id */, + "160223" /* date */ , + "196" /* instrument name*/, + 2371 /* run number */, + 3 /** RunInfo version */, + model::run::flowcell_layout(8 /*lane count*/, + 2 /*surface count*/, + 2 /*swath count */, + 28 /*tile count */, + 1 /* sections per lane */, + 1 /* lanes per section*/, + util::to_vector(tiles) /* Tiles */, + constants::FourDigit, + "H7MF5BBXX" /* flowcell id */), + util::to_vector(channels) /*Image channels */, + model::run::image_dimensions(/*Width*/ 3188, /*Height*/ 4826), + reads); + } + /** Generate expected run info relating to file representation + * + * @param expected_run_info destination run info + */ + static void create_expected(model::run::info& expected_run_info) + { + const model::run::read_info reads[] = { + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + model::run::read_info(1, 1, 151, false), + model::run::read_info(2, 152, 302, false) + }; + create_expected(expected_run_info, util::to_vector(reads)); + } + }; + + /** Provide an example MiniSeq RunInfo v4 + */ + struct miniseq_run_info + { + /** Generate a file representation of the RunInfo in a string + * + * @param data destination string + */ + static void create_string(std::string& data) + { + data= "\n" + "\n" + " \n" + " 000H02MNK\n" + " ML-P2-01\n" + " 160822\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " 1_11102\n" + " 1_21102\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " Red\n" + " Green\n" + " \n" + " \n" + ""; + } + /** Generate a standard runinfo with an arbitrary read structure + * + * @param expected_run_info destination run info + * @param reads arbitrary read structure + */ + static void create_expected(model::run::info& expected_run_info, const std::vector& reads) + { + std::string tiles[] = { + "1_11102", + "1_21102" + }; + const std::string channels[] = {"red", "green"}; + expected_run_info = model::run::info("160822_ML-P2-01_0042_A000H02MNK" /* run id */, + "160822" /* date */ , + "ML-P2-01" /* instrument name*/, + 42 /* run number */, + 4 /** RunInfo version */, + model::run::flowcell_layout(1 /*lane count*/, + 2 /*surface count*/, + 3 /*swath count */, + 10 /*tile count */, + 1 /* sections per lane */, + 1 /* lanes per section*/, + util::to_vector(tiles) /* Tiles */, + constants::FiveDigit, + "000H02MNK" /* flowcell id */), + util::to_vector(channels) /*Image channels */, + model::run::image_dimensions(/*Width*/ 2592, /*Height*/ 1944), + reads); + } + /** Generate expected run info relating to file representation + * + * @param expected_run_info destination run info + */ + static void create_expected(model::run::info& expected_run_info) + { + const model::run::read_info reads[] = { + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + model::run::read_info(1, 1, 150, false), + model::run::read_info(2, 151, 158, true), + model::run::read_info(3, 159, 166, true), + model::run::read_info(4, 167, 316, false) + }; + create_expected(expected_run_info, util::to_vector(reads)); + } + }; + + + /** Provide an example NextSeq 550 RunInfo v4 + */ + struct nextseq_550_run_info + { + /** Generate a file representation of the RunInfo in a string + * + * @param data destination string + */ + static void create_string(std::string& data) + { + data= "\n" + "\n" + " \n" + " HW3YTBGXX\n" + " NS500179\n" + " 160630\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " 1_11101\n" + " 1_21101\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " Red\n" + " Green\n" + " \n" + " \n" + ""; + } + /** Generate a standard runinfo with an arbitrary read structure + * + * @param expected_run_info destination run info + * @param reads arbitrary read structure + */ + static void create_expected(model::run::info& expected_run_info, const std::vector& reads) + { + std::string tiles[] = { + "1_11101", + "1_21101" + }; + const std::string channels[] = {"red", "green"}; + expected_run_info = model::run::info("160630_NS500179_0099_AHW3YTBGXX" /* run id */, + "160630" /* date */ , + "NS500179" /* instrument name*/, + 99 /* run number */, + 4 /** RunInfo version */, + model::run::flowcell_layout(4 /*lane count*/, + 2 /*surface count*/, + 3 /*swath count */, + 12 /*tile count */, + 3 /* sections per lane */, + 2 /* lanes per section*/, + util::to_vector(tiles) /* Tiles */, + constants::FiveDigit, + "HW3YTBGXX" /* flowcell id */), + util::to_vector(channels) /*Image channels */, + model::run::image_dimensions(/*Width*/ 2592, /*Height*/ 1944), + reads); + } + /** Generate expected run info relating to file representation + * + * @param expected_run_info destination run info + */ + static void create_expected(model::run::info& expected_run_info) + { + const model::run::read_info reads[] = { + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + model::run::read_info(1, 1, 101, false), + model::run::read_info(2, 102, 107, true), + model::run::read_info(3, 108, 208, false) + }; + create_expected(expected_run_info, util::to_vector(reads)); + } + }; + + + /** Provide an example NovaSeq RunInfo v5 + */ + struct novaseq_run_info + { + /** Generate a file representation of the RunInfo in a string + * + * @param data destination string + */ + static void create_string(std::string& data) + { + data= "\n" + "\n" + "\t\n" + "\t\tYYYYYYYY\n" + "\t\tXXXXXX\n" + "\t\t2/8/2017 4:25:41 PM\n" + "\t\t\n" + "\t\t\t\n" + "\t\t\t\n" + "\t\t\t\n" + "\t\t\t\n" + "\t\t\n" + "\t\t\n" + "\t\t\t\n" + "\t\t\t\t\n" + "\t\t\t\t\t1_2101\n" + "\t\t\t\t\t1_2102\n" + "\t\t\t\t\n" + "\t\t\t\n" + "\t\t\n" + "\t\t\n" + "\t\t\n" + "\t\t\n" + "\t\t\tRED\n" + "\t\t\tGREEN\n" + "\t\t\n" + "\t\n" + ""; + } + /** Generate a standard runinfo with an arbitrary read structure + * + * @param expected_run_info destination run info + * @param reads arbitrary read structure + */ + static void create_expected(model::run::info& expected_run_info, const std::vector& reads) + { + std::string tiles[] = { + "1_2101", + "1_2102" + }; + const std::string channels[] = {"red", "green"}; + expected_run_info = model::run::info("170208_XXXXXX_0021_YYYYYYYY" /* run id */, + "2/8/2017 4:25:41 PM" /* date */ , + "XXXXXX" /* instrument name*/, + 21 /* run number */, + 0 /** RunInfo version */, + model::run::flowcell_layout(2 /*lane count*/, + 2 /*surface count*/, + 4 /*swath count */, + 88 /*tile count */, + 1 /* sections per lane */, + 1 /* lanes per section*/, + util::to_vector(tiles) /* Tiles */, + constants::FourDigit, + "YYYYYYYY" /* flowcell id */), + util::to_vector(channels) /*Image channels */, + model::run::image_dimensions(/*Width*/ 3200, /*Height*/ 3607), + reads); + } + /** Generate expected run info relating to file representation + * + * @param expected_run_info destination run info + */ + static void create_expected(model::run::info& expected_run_info) + { + const model::run::read_info reads[] = { + /*Number, IsIndexedRead, CycleStart, CycleEnd*/ + model::run::read_info(1, 1, 151, false), + model::run::read_info(2, 152, 159, true), + model::run::read_info(3, 160, 167, true), + model::run::read_info(4, 168, 318, false) + }; + create_expected(expected_run_info, util::to_vector(reads)); + } + }; + +}}} + + diff --git a/tools/build_cov_test.sh b/tools/build_cov_test.sh index 7bdffdf38..06d6a29d0 100644 --- a/tools/build_cov_test.sh +++ b/tools/build_cov_test.sh @@ -19,7 +19,8 @@ set -e -source_dir="../" +source_dir="${PWD}" +build_dir="${PWD}/build" build_type="Debug" build_param="" @@ -27,15 +28,19 @@ if [ ! -z $1 ] ; then build_type=$1 fi if [ ! -z $2 ] ; then - build_param="-DGTEST_ROOT=$2 -DGMOCK_ROOT=$2 -DNUNIT_ROOT=$2/NUnit-2.6.4" + pushd $2 > /dev/null + build_path=`pwd` + popd > /dev/null + build_param="-DGTEST_ROOT=$build_path -DGMOCK_ROOT=$build_path -DNUNIT_ROOT=$build_path/NUnit-2.6.4" fi if [ ! -z $3 ] ; then cov_path=$3 fi echo "##teamcity[blockOpened name='Configure $build_type']" -mkdir build_cov_${build_type} -cd build_cov_${build_type} +rm -fr $build_dir +mkdir $build_dir +cd $build_dir echo "cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param" cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param echo "##teamcity[blockClosed name='Configure $build_type']" @@ -53,5 +58,5 @@ $cov_path/cov-commit-defects --dir cov --host ussd-prd-cove01 --stream InterOp_M echo "##teamcity[blockClosed name='Coverity Upload $build_type']" cd .. - +rm -fr $build_dir diff --git a/tools/build_test.bat b/tools/build_test.bat index 2714e83a1..34a711552 100644 --- a/tools/build_test.bat +++ b/tools/build_test.bat @@ -18,7 +18,9 @@ rem ---------------------------------------------------------------------------- rem MinGW Build Test Script rem -------------------------------------------------------------------------------------------------------------------- -set SOURCE_DIR=..\ +set SOURCE_DIR=%CD% +set BUILD_DIR=%SOURCE_DIR%\build +set DIST_DIR=%SOURCE_DIR%\dist set BUILD_PARAM= set BUILD_TYPE=Debug set COMPILER=msvc @@ -28,11 +30,13 @@ if NOT "%1" == "" ( set BUILD_TYPE=%1 ) set BUILD_PATH=%2% +pushd %BUILD_PATH% +set BUILD_PATH=%CD% +popd if NOT "%2" == "" ( set BUILD_PARAM=-DGTEST_ROOT=%BUILD_PATH% -DGMOCK_ROOT=%BUILD_PATH% -DNUNIT_ROOT=%BUILD_PATH%/NUnit-2.6.4 ) -set BUILD_PATH=%3% if NOT "%3" == "" ( set BUILD_PARAM=%BUILD_PARAM% -DBUILD_NUMBER=%3% ) @@ -40,12 +44,18 @@ if NOT "%4" == "" ( set COMPILER=%4% ) +if exist %BUILD_DIR% rd /s /q %BUILD_DIR% +if exist %DIST_DIR% rd /s /q %DIST_DIR% +mkdir %BUILD_DIR% +cd %BUILD_DIR% + + +set BUILD_PARAM=%BUILD_PARAM% -DPACKAGE_OUTPUT_FILE_PREFIX=%DIST_DIR% -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -DCMAKE_INSTALL_PREFIX=%DIST_DIR% + if "%COMPILER%" == "mingw" ( echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% MinGW'] -mkdir build_mingw_%BUILD_TYPE% -cd build_mingw_%BUILD_TYPE% -echo cmake %SOURCE_DIR% -G"MinGW Makefiles" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% -cmake %SOURCE_DIR% -G"MinGW Makefiles" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% -DCMAKE_INSTALL_PREFIX=../usr +echo cmake %SOURCE_DIR% -G"MinGW Makefiles" %BUILD_PARAM% +cmake %SOURCE_DIR% -G"MinGW Makefiles" %BUILD_PARAM% -DENABLE_PYTHON=OFF if !errorlevel! neq 0 exit /b !errorlevel! echo ##teamcity[blockClosed name='Configure %BUILD_TYPE% MinGW'] set MT=-j8 @@ -53,10 +63,8 @@ set INSTALL=install ) if "%COMPILER%" == "msvc" ( echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] -mkdir build_vs2015_x64_%BUILD_TYPE% -cd build_vs2015_x64_%BUILD_TYPE% -echo cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% -cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% +echo cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" %BUILD_PARAM% +cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" %BUILD_PARAM% if !errorlevel! neq 0 exit /b !errorlevel! echo ##teamcity[blockClosed name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] ) @@ -67,16 +75,25 @@ cmake --build . --config %BUILD_TYPE% -- %MT% if %errorlevel% neq 0 exit /b %errorlevel% echo ##teamcity[blockClosed name='Build %BUILD_TYPE% %COMPILER%'] -echo ##teamcity[blockOpened name='Test %BUILD_TYPE% MinGW'] +echo ##teamcity[blockOpened name='Test %BUILD_TYPE% %COMPILER%'] cmake --build . --config %BUILD_TYPE% --target check -- %MT% if %errorlevel% neq 0 exit /b %errorlevel% -echo ##teamcity[blockClosed name='Test %BUILD_TYPE% MinGW'] +echo ##teamcity[blockClosed name='Test %BUILD_TYPE% %COMPILER%'] -echo ##teamcity[blockOpened name='Install %BUILD_TYPE% MinGW'] +echo ##teamcity[blockOpened name='Install %BUILD_TYPE% %COMPILER%'] cmake --build . --config Release --target %INSTALL% -- %MT% if %errorlevel% neq 0 exit /b %errorlevel% -echo ##teamcity[blockClosed name='Install %BUILD_TYPE% MinGW'] +echo ##teamcity[blockClosed name='Install %BUILD_TYPE% %COMPILER%'] -cd .. +if "%COMPILER%" == "msvc" ( +echo ##teamcity[blockOpened name='NuPack %BUILD_TYPE% %COMPILER%'] +echo %BUILD_PATH%\nuget pack %BUILD_DIR%\src\ext\csharp\package.nuspec +%BUILD_PATH%\nuget pack %BUILD_DIR%\src\ext\csharp\package.nuspec -OutputDirectory %DIST_DIR% +echo ##teamcity[blockClosed name='NuPack %BUILD_TYPE% %COMPILER%'] + +) +cd %SOURCE_DIR% +rd /s /q %BUILD_DIR% +if exist %BUILD_DIR% rd /s /q %BUILD_DIR% diff --git a/tools/build_test.sh b/tools/build_test.sh index 40ef144c9..dda0f301d 100644 --- a/tools/build_test.sh +++ b/tools/build_test.sh @@ -18,29 +18,38 @@ set -e -source_dir="../" +source_dir="${PWD}" +dist_dir="${PWD}/dist" +build_dir="${PWD}/build" build_type="Debug" build_param="" if [ ! -z $1 ] ; then build_type=$1 fi - if [ ! -z $2 ] ; then - build_param="-DGTEST_ROOT=$2 -DGMOCK_ROOT=$2 -DNUNIT_ROOT=$2/NUnit-2.6.4" + pushd $2 > /dev/null + build_path=`pwd` + popd > /dev/null + build_param="-DGTEST_ROOT=$build_path -DGMOCK_ROOT=$build_path -DNUNIT_ROOT=$build_path/NUnit-2.6.4" fi +build_param="$build_param -DCMAKE_INSTALL_PREFIX=$dist_dir" if [ -e /opt/rh/devtoolset-2/root/usr/bin/g++ ] ; then - export CXX=/usr/bin/g++ - export CC=/usr/bin/gcc + build_type="Release" + export CXX=/opt/rh/devtoolset-2/root/usr/bin/g++ + export CC=/opt/rh/devtoolset-2/root/usr/bin/gcc + echo "Found GCC4.8 dev" fi echo "##teamcity[blockOpened name='Configure $build_type']" -mkdir build_${build_type} -cd build_${build_type} -echo "cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param" -cmake $source_dir -DCMAKE_BUILD_TYPE=$build_type $build_param -DCMAKE_INSTALL_PREFIX=../usr +rm -fr $build_dir +rm -fr $dist_dir +mkdir $build_dir +cd $build_dir +echo "cmake $source_dir $build_param -DCMAKE_BUILD_TYPE=$build_type" +cmake $source_dir $build_param -DCMAKE_BUILD_TYPE=$build_type echo "##teamcity[blockClosed name='Configure $build_type']" echo "##teamcity[blockOpened name='Build $build_type']" @@ -53,8 +62,9 @@ echo "##teamcity[blockClosed name='Test $build_type']" echo "##teamcity[blockOpened name='Install $build_type']" rm -f CMakeCache.txt -cmake $source_dir -DCMAKE_BUILD_TYPE=Release $build_param -DCMAKE_INSTALL_PREFIX=../usr +cmake $source_dir -DCMAKE_BUILD_TYPE=Release $build_param cmake --build . --target install -- -j4 echo "##teamcity[blockClosed name='Install $build_type']" cd .. +rm -fr $build_dir diff --git a/tools/package.bat b/tools/package.bat index af92a4f22..0de9c8a80 100644 --- a/tools/package.bat +++ b/tools/package.bat @@ -17,19 +17,27 @@ rem ---------------------------------------------------------------------------- rem MinGW Build Test Script rem -------------------------------------------------------------------------------------------------------------------- -set SOURCE_DIR=..\ +set SOURCE_DIR=%CD% +set BUILD_DIR=%SOURCE_DIR%\build +set DIST_DIR=%SOURCE_DIR%\dist set BUILD_PARAM= set BUILD_TYPE=Release set BUILD_PATH=%1% +pushd %BUILD_PATH% +set BUILD_PATH=%CD% +popd if NOT "%1" == "" ( set BUILD_PARAM=-DGTEST_ROOT=%BUILD_PATH% -DJUNIT_ROOT=%BUILD_PATH% -DGMOCK_ROOT=%BUILD_PATH% -DNUNIT_ROOT=%BUILD_PATH%/NUnit-2.6.4 ) +set BUILD_PARAM=%BUILD_PARAM% -DPACKAGE_OUTPUT_FILE_PREFIX=%DIST_DIR% echo ##teamcity[blockOpened name='Configure %BUILD_TYPE% Visual Studio 2015 Win64'] -mkdir build_vs2015_x64_%BUILD_TYPE% -cd build_vs2015_x64_%BUILD_TYPE% +if exist %BUILD_DIR% rd /s /q %BUILD_DIR% +if exist %DIST_DIR% rd /s /q %DIST_DIR% +mkdir %BUILD_DIR% +cd %BUILD_DIR% echo cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% cmake %SOURCE_DIR% -G"Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=%BUILD_TYPE% %BUILD_PARAM% if %errorlevel% neq 0 exit /b %errorlevel% @@ -55,3 +63,12 @@ cmake --build . --target nuspec --config Release -- /M if %errorlevel% neq 0 exit /b %errorlevel% echo ##teamcity[blockClosed name='NuSpec Release Visual Studio 2015 Win64'] + +echo "##teamcity[blockOpened name='NuPack Visual Studio 2015 Win64']" +echo %BUILD_PATH%\nuget pack %BUILD_DIR%\src\ext\csharp\package.nuspec +%BUILD_PATH%\nuget pack %BUILD_DIR%\src\ext\csharp\package.nuspec -OutputDirectory %DIST_DIR% +echo "##teamcity[blockClosed name='NuPack Visual Studio 2015 Win64']" + +cd %SOURCE_DIR% +rd /s /q %BUILD_DIR% +if exist %BUILD_DIR% rd /s /q %BUILD_DIR% diff --git a/tools/package.sh b/tools/package.sh index 50286a0ba..28cf81320 100644 --- a/tools/package.sh +++ b/tools/package.sh @@ -21,11 +21,17 @@ set -e source_dir="../" build_param="" +root_dir=${PWD} +dist_dir="${PWD}/dist" +build_dir="${PWD}/build" build_type="Linux" if [ ! -z $1 ] ; then - build_param="-DJUNIT_ROOT=$1 -DGTEST_ROOT=$1 -DGMOCK_ROOT=$1 -DNUNIT_ROOT=$1/NUnit-2.6.4" + pushd $1 > /dev/null + build_path=`pwd` + popd > /dev/null + build_param="-DGTEST_ROOT=$build_path -DGMOCK_ROOT=$build_path -DNUNIT_ROOT=$build_path/NUnit-2.6.4" fi if [ ! -z $2 ] ; then @@ -36,16 +42,21 @@ if [ ! -z $3 ] ; then build_param="$build_param -DPACKAGE_SUFFIX=$3" fi +build_param="$build_param -DCMAKE_BUILD_TYPE=Release -DPACKAGE_OUTPUT_FILE_PREFIX=$dist_dir" + if [ -e /opt/rh/devtoolset-2/root/usr/bin/g++ ] ; then export CXX=/opt/rh/devtoolset-2/root/usr/bin/g++ export CC=/opt/rh/devtoolset-2/root/usr/bin/gcc + echo "Found GCC4.8 dev" fi echo "##teamcity[blockOpened name='Configure $build_type']" -mkdir build_${build_type} -cd build_${build_type} -echo "cmake $source_dir -DCMAKE_BUILD_TYPE=Release $build_param" -cmake $source_dir -DCMAKE_BUILD_TYPE=Release $build_param +rm -fr $dist_dir +rm -fr $build_dir +mkdir $build_dir +cd $build_dir +echo "cmake $source_dir $build_param " +cmake $source_dir $build_param echo "##teamcity[blockClosed name='Configure $build_type']" echo "##teamcity[blockOpened name='Test $build_type']" @@ -64,8 +75,10 @@ echo "##teamcity[blockOpened name='NuSpec Creation $build_type']" cmake --build . --target nuspec -- -j 8 echo "##teamcity[blockClosed name='NuSpec Creation $build_type']" +cd $dist_dir echo "##teamcity[blockOpened name='NuPack $build_type']" -nuget pack src/ext/csharp/package.nuspec +nuget pack ${build_dir}/src/ext/csharp/package.nuspec echo "##teamcity[blockClosed name='NuPack $build_type']" -cd .. +cd $root_dir +rm -fr $build_dir diff --git a/tools/regression_test.ps1 b/tools/regression_test.ps1 index b8f9e22d3..052ae6688 100644 --- a/tools/regression_test.ps1 +++ b/tools/regression_test.ps1 @@ -26,20 +26,30 @@ param( ) $build_param="" -if($lib_path) { $build_param="-DGTEST_ROOT=$lib_path -DJUNIT_ROOT=$lib_path -DGMOCK_ROOT=$lib_path -DNUNIT_ROOT=$lib_path/NUnit-2.6.4"} +if($lib_path) +{ +$lib_path=(resolve-path $lib_path).path +$build_param="-DGTEST_ROOT=$lib_path -DJUNIT_ROOT=$lib_path -DGMOCK_ROOT=$lib_path -DNUNIT_ROOT=$lib_path/NUnit-2.6.4" +} # -------------------------------------------------------------------------------------------------------------------- Write-Host "##teamcity[blockOpened name='Configure $config $generator']" -new-item build_vs2015_x64_$config -itemtype directory -set-location -path build_vs2015_x64_$config +if(Test-Path -Path build){ + Remove-Item build -Force -Recurse +} +new-item build -itemtype directory +set-location -path build Write-Host "cmake $source_path -G $generator -DCMAKE_BUILD_TYPE=$config $build_param" -cmake $source_path -G $generator -DCMAKE_BUILD_TYPE=$config $build_param +cmake $source_path -G $generator -DCMAKE_BUILD_TYPE=$config $build_param.Split(" ") $test_code=$lastexitcode Write-Host "##teamcity[blockClosed name='Configure $config $generator']" if ($test_code -ne 0) { cd .. Write-Host "##teamcity[buildStatus status='FAILURE' text='Configure Failed!']" + if(Test-Path -Path build){ + Remove-Item build -Force -Recurse + } exit $test_code } @@ -53,6 +63,9 @@ if ($test_code -ne 0) { cd .. Write-Host "##teamcity[buildStatus status='FAILURE' text='Build Failed!']" + if(Test-Path -Path build){ + Remove-Item build -Force -Recurse + } exit 1 } @@ -75,6 +88,9 @@ if($rebaseline) { cd .. Write-Host "##teamcity[buildStatus status='FAILURE' text='Rebaseline failed!']" + if(Test-Path -Path build){ + Remove-Item build -Force -Recurse + } exit 1 } } @@ -87,9 +103,14 @@ if ($test_code -ne 0) { cd .. Write-Host "##teamcity[buildStatus status='FAILURE' text='Not all regression tests passed!']" + if(Test-Path -Path build){ + Remove-Item build -Force -Recurse + } exit 1 } cd .. - +if(Test-Path -Path build){ + Remove-Item build -Force -Recurse +}