From 997ad6f470b578b6dfef2911a8f8e9ab5ba4bb5a Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Wed, 17 Oct 2018 17:33:19 +0100 Subject: [PATCH 1/9] Adding `species` and `genomes` endpoints, adding `genomes` page. Refactoring column_formatter to allow dropdown lists without links. Updating Eve to v0.8. --- etc/column_mappings.yaml | 47 ++++++- etc/schema.yaml | 20 +++ reporting_app/__init__.py | 21 +++ reporting_app/static/column_formatter.js | 121 +++++++++++------- .../static/stylesheets/reporting_app.css | 1 + reporting_app/templates/base.html | 1 + requirements.txt | 5 +- rest_api/settings.py | 8 +- tests/js/test_column_formatter.js | 10 ++ tests/test_reporting_app.py | 3 + 10 files changed, 180 insertions(+), 57 deletions(-) diff --git a/etc/column_mappings.yaml b/etc/column_mappings.yaml index bed2abd6..aea4097b 100644 --- a/etc/column_mappings.yaml +++ b/etc/column_mappings.yaml @@ -114,6 +114,20 @@ column_def: trim_r2: { data: 'aggregated.trim_r2', title: 'Read 2 Max length', visible: 'false'} tiles_filtered: { data: 'aggregated.tiles_filtered', title: 'Tile removed', visible: 'false'} family_id: { data: 'Family ID', title: 'Family ID (SGP)', visible: 'false'} + species_name: { data: 'name', title: 'Name' } + species_default_genome: { data: 'default_version', title: 'Default genome version' } + species_taxid: { data: 'taxid', title: 'Taxid' } + genome_assembly: { data: 'assembly_name', title: 'Assembly name' } + genome_species: { data: 'species', title: 'Species' } + genome_data_source: { data: 'data_source', title: 'Data source' } + genome_tools_used: { data: 'tools_used', title: 'Tools used', visible: 'false' } + genome_chromosome_count: { data: 'chromosome_count', title: 'Chromosomes' } + genome_size: { data: 'genome_size', title: 'Genome size' } + genome_goldenpath: { data: 'goldenpath', title: 'GoldenPath' } + genome_projects: { data: 'project_whitelist', title: 'Allowed projects', visible: 'false' } + genome_comments: { data: 'comments', title: 'Comments', visible: 'false' } + genome_analyses: { data: 'analyses_supported', title: 'Analyses supported' } + # Datatables tables definition # the remaining top level entries each define a table with a list of column @@ -343,10 +357,29 @@ sample_run_elements: active_runs: - - run_id - - instrument_id - - run_status - - created_date - - cst_date - - project_ids - - sample_ids + - run_id + - instrument_id + - run_status + - created_date + - cst_date + - project_ids + - sample_ids + + +species: + - species_name + - species_default_genome + - species_taxid + + +genomes: + - genome_assembly + - genome_species + - genome_data_source + - genome_chromosome_count + - genome_size + - genome_goldenpath + - genome_analyses + - genome_tools_used + - genome_projects + - genome_comments diff --git a/etc/schema.yaml b/etc/schema.yaml index 99178664..e0fe6d84 100644 --- a/etc/schema.yaml +++ b/etc/schema.yaml @@ -251,6 +251,7 @@ analysis_driver_stages: date_finished: { type: 'datetime', nullable: True } exit_status: { type: 'integer', nullable: True } + actions: action_id: { type: 'string', required: True, unique: True } action_type: { type: 'string', required: True } @@ -260,3 +261,22 @@ actions: action_info: { type: 'dict' } +species: + name: { type: 'string', required: True, unique: True } + genomes: { type: 'list', schema: { type: 'string', data_relation: { resource: 'genomes', field: 'assembly_name', embeddable: True } } } + default_version: { type: 'string' } + taxid: { type: 'string' } + + +genomes: + assembly_name: { type: 'string', required: True, unique: True } + species: { type: 'string' } + data_files: { type: 'dict', schema: { fasta: { type: 'string' }, variation: { type: 'string' } } } + data_source: { type: 'string' } + tools_used: { type: 'dict' } + chromosome_count: { type: 'integer' } + genome_size: { type: 'integer' } + goldenpath: { type: 'integer' } + project_whitelist: { type: 'list', schema: { type: 'string' } } + comments: { type: 'string' } + analyses_supported: { type: 'list', schema: { type: 'string', allowed: ['qc', 'variant_calling', 'bcbio'] } } diff --git a/reporting_app/__init__.py b/reporting_app/__init__.py index dda2f4ff..81b2f3eb 100644 --- a/reporting_app/__init__.py +++ b/reporting_app/__init__.py @@ -453,3 +453,24 @@ def project_status_reports(prj_status): table_foot='sum_row_per_column' ) ) + + +@app.route('/genomes') +@flask_login.login_required +def genome_page(): + return render_template( + 'untabbed_datatables.html', + 'Genomes', + tables=[ + util.datatable_cfg( + 'Available species', + 'species', + util.construct_url('species', all_pages=True) + ), + util.datatable_cfg( + 'Installed genomes', + 'genomes', + util.construct_url('genomes', all_pages=True) + ) + ] + ) diff --git a/reporting_app/static/column_formatter.js b/reporting_app/static/column_formatter.js index 84a6832e..90250922 100644 --- a/reporting_app/static/column_formatter.js +++ b/reporting_app/static/column_formatter.js @@ -8,7 +8,7 @@ function render_data(data, type, row, meta, fmt) { return null; } if (!fmt) { - return '
' + data + '
'; + fmt = {}; } if (fmt['name']) { data = function_map[fmt['name']](data, fmt) @@ -17,61 +17,96 @@ function render_data(data, type, row, meta, fmt) { } -function string_formatter(data, fmt, row){ - var formatted_data = data; - - if (fmt['type'] == 'percentage') { - formatted_data = Humanize.toFixed(formatted_data, 1) + '%'; - }if (fmt['type'] == 'ratio_percentage') { - formatted_data = Humanize.toFixed(formatted_data * 100, 1) + '%'; - } else if (fmt['type'] == 'int') { - formatted_data = Humanize.intComma(formatted_data); - } else if (fmt['type'] == 'float') { - formatted_data = Humanize.formatNumber(formatted_data, 2); - } else if (fmt['type'] == 'date') { - formatted_data = moment(new Date(formatted_data)).format('YYYY-MM-DD'); - } else if (fmt['type'] == 'datetime') { - formatted_data = moment(new Date(formatted_data)).format('YYYY-MM-DD HH:mm:ss'); - } - if (fmt['link']) { - if (fmt['link_format_function']){ - formatted_link = function_map[fmt['link_format_function']](data, fmt); - } - else{ - formatted_link = data; - } - if (data instanceof Array && data.length > 1 || data != formatted_link) { - data.sort(); - formatted_data = '
' + formatted_link + '
'; - for (var i=0, tot=data.length; i < tot; i++){ - formatted_data = formatted_data.concat('' + data[i] + ''); - } - formatted_data = formatted_data.concat('
') +function string_formatter(cell_data, fmt, row){ + if (cell_data instanceof Array) { + cell_data.sort(); + } else if (cell_data instanceof Object) { + var _cell_data = []; + for (k in cell_data) { + _cell_data.push(k + ': ' + cell_data[k]); } - else if (data instanceof Array && data.length == 1){ - formatted_data = '' + data[0] + ''; - } - else { - formatted_data = '' + data + ''; + cell_data = _cell_data; + } else { + cell_data = [cell_data]; + } + + var use_dropdown = false; + if (cell_data.length > 1) { + use_dropdown = true; + } + var formatted_data = []; + + for (var i=0, tot=cell_data.length; i' + data + ''; } + var min, max; - if (fmt['min']){ + if (fmt['min']) { min = resolve_min_max_value(row, fmt['min']) } - if (fmt['max']){ + if (fmt['max']) { max = resolve_min_max_value(row, fmt['max']) } if (min && data < min) { - formatted_data = '
' + formatted_data + '
'; + _formatted_data = '
' + _formatted_data + '
'; } else if (max && !isNaN(max) && data > max) { - formatted_data = '
' + formatted_data + '
'; + _formatted_data = '
' + _formatted_data + '
'; } else if (max && data > max) { - formatted_data = '
' + formatted_data + '
'; + _formatted_data = '
' + _formatted_data + '
'; + } + + formatted_data.push(_formatted_data); + + } + + if (use_dropdown) { + var dropdown = document.createElement('div'); + dropdown.className = 'dropdown'; + + var dropbtn = document.createElement('div'); + dropbtn.className = 'dropbtn'; + if (fmt['link_format_function']) { + dropbtn.innerHTML = function_map[fmt['link_format_function']](cell_data, fmt); + } else { + dropbtn.innerHTML = cell_data; + } + + var dropdown_content = document.createElement('div'); + dropdown_content.className = 'dropdown-content'; + + var div; + for (var i=0, tot=formatted_data.length; i'; - return formatted_data; + return '
' + formatted_data + '
'; } diff --git a/reporting_app/static/stylesheets/reporting_app.css b/reporting_app/static/stylesheets/reporting_app.css index e1aed946..65ef2483 100644 --- a/reporting_app/static/stylesheets/reporting_app.css +++ b/reporting_app/static/stylesheets/reporting_app.css @@ -17,6 +17,7 @@ } .dropdown { + padding: 2px; //display: inline-block; } diff --git a/reporting_app/templates/base.html b/reporting_app/templates/base.html index 3bc629d1..dfebc0eb 100644 --- a/reporting_app/templates/base.html +++ b/reporting_app/templates/base.html @@ -77,6 +77,7 @@
  • Project Status - All
    • +
  • Genomes
  • Charts
  • {{ current_user.username }} diff --git a/requirements.txt b/requirements.txt index 3e7d9237..518cce02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,4 @@ -Cerberus==0.9.2 -Eve==0.7.4 -Events==0.2.1 -Flask==0.10.1 +Eve==0.8 Flask-Cors==2.1.2 Flask-Login==0.3.2 Flask-PyMongo==0.4.1 diff --git a/rest_api/settings.py b/rest_api/settings.py index aa251d9b..24f86579 100644 --- a/rest_api/settings.py +++ b/rest_api/settings.py @@ -15,7 +15,7 @@ for endpoint in ( 'runs', 'lanes', 'run_elements', 'unexpected_barcodes', 'projects', 'samples', 'analysis_driver_procs', - 'analysis_driver_stages', 'actions' + 'analysis_driver_stages', 'actions', 'species', 'genomes' ) } @@ -26,7 +26,9 @@ ('samples', 'sample_id'), ('analysis_driver_procs', 'proc_id'), ('analysis_driver_stages', 'stage_id'), - ('actions', 'action_id') + ('actions', 'action_id'), + ('species', 'name'), + ('genomes', 'assembly_name') ): DOMAIN[endpoint]['id_field'] = id_field @@ -36,7 +38,7 @@ MONGO_DBNAME = cfg['db_name'] ITEMS = 'data' -XML = False +RENDERERS = ['eve.render.JSONRenderer'] PAGINATION = True PAGINATION_LIMIT = 100000 diff --git a/tests/js/test_column_formatter.js b/tests/js/test_column_formatter.js index f3097594..4ef35284 100644 --- a/tests/js/test_column_formatter.js +++ b/tests/js/test_column_formatter.js @@ -77,4 +77,14 @@ QUnit.test('string_formatter', function(assert) { '
    6
    ' ); + assert.equal( + string_formatter(['this', 'that'], {}, row), + '
    that,this
    that
    this
    ' + ); + + assert.equal( + string_formatter(['1', '2'], {'link': '/a_local_page/'}, row), + '
    1,2
    ' + ); + }); diff --git a/tests/test_reporting_app.py b/tests/test_reporting_app.py index aed93640..f373b22f 100644 --- a/tests/test_reporting_app.py +++ b/tests/test_reporting_app.py @@ -330,3 +330,6 @@ def test_plotting_report(self): def test_project_status_report(self): self._test_render_template('/project_status/') + + def test_genome_page(self): + self._test_render_template('/genomes') From 4ee6eea13475ca7bd869af755117ff0b66e57a0a Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Fri, 19 Oct 2018 16:47:22 +0100 Subject: [PATCH 2/9] Adding available coverage/yield configs, adding approximate_genome_size, and --- etc/example_reporting.yaml | 7 ++++ etc/schema.yaml | 32 +++++++++-------- rest_api/aggregation/database_hooks.py | 36 +++++++++++++++++++ .../test_aggregation/test_database_hooks.py | 22 ++++++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/etc/example_reporting.yaml b/etc/example_reporting.yaml index bb7a0b61..e2876a6e 100644 --- a/etc/example_reporting.yaml +++ b/etc/example_reporting.yaml @@ -50,3 +50,10 @@ rest_app: baseuri: http://clarity.com username: user password: pass + + available_coverages: [1, 2, 5, 10, 20] + available_yields: # keys are required yields, and values are associated required yield q30s + 5: 4 + 10: 8 + 20: 16 + 40: 32 diff --git a/etc/schema.yaml b/etc/schema.yaml index e0fe6d84..9420df7c 100644 --- a/etc/schema.yaml +++ b/etc/schema.yaml @@ -242,6 +242,7 @@ analysis_driver_procs: status: { type: 'string', allowed: ['force_ready', 'processing', 'finished', 'failed', 'aborted', 'reprocess', 'deleted', 'resume'] } stages: { type: 'list', schema: { type: 'string', data_relation: { resource: 'analysis_driver_stages', field: 'stage_id', embeddable: True } } } data_source: { type: 'list', required: False, schema: { type: 'string' } } + genome_used: { type: 'string' } # TODO: this should be stored in samples when doing #112 analysis_driver_stages: stage_id: { type: 'string', required: True, unique: True } @@ -262,21 +263,22 @@ actions: species: - name: { type: 'string', required: True, unique: True } - genomes: { type: 'list', schema: { type: 'string', data_relation: { resource: 'genomes', field: 'assembly_name', embeddable: True } } } - default_version: { type: 'string' } - taxid: { type: 'string' } + name: { type: 'string', required: True, unique: True } + genomes: { type: 'list', schema: { type: 'string', data_relation: { resource: 'genomes', field: 'assembly_name', embeddable: True } } } + default_version: { type: 'string' } + taxid: { type: 'string' } + approximate_genome_size: { type: 'integer' } genomes: - assembly_name: { type: 'string', required: True, unique: True } - species: { type: 'string' } - data_files: { type: 'dict', schema: { fasta: { type: 'string' }, variation: { type: 'string' } } } - data_source: { type: 'string' } - tools_used: { type: 'dict' } - chromosome_count: { type: 'integer' } - genome_size: { type: 'integer' } - goldenpath: { type: 'integer' } - project_whitelist: { type: 'list', schema: { type: 'string' } } - comments: { type: 'string' } - analyses_supported: { type: 'list', schema: { type: 'string', allowed: ['qc', 'variant_calling', 'bcbio'] } } + assembly_name: { type: 'string', required: True, unique: True } + species: { type: 'string' } + data_files: { type: 'dict', schema: { fasta: { type: 'string' }, variation: { type: 'string' } } } + data_source: { type: 'string' } + tools_used: { type: 'dict' } + chromosome_count: { type: 'integer' } + genome_size: { type: 'integer' } + goldenpath: { type: 'integer' } + project_whitelist: { type: 'list', schema: { type: 'string' } } + comments: { type: 'string' } + analyses_supported: { type: 'list', schema: { type: 'string', allowed: ['qc', 'variant_calling', 'bcbio'] } } diff --git a/rest_api/aggregation/database_hooks.py b/rest_api/aggregation/database_hooks.py index a99933a1..9fa85e3e 100644 --- a/rest_api/aggregation/database_hooks.py +++ b/rest_api/aggregation/database_hooks.py @@ -1,6 +1,7 @@ import statistics from rest_api.aggregation.server_side.expressions import * from rest_api.aggregation.database_side import db +from rest_api import cfg def deep_dict_update(d, u): @@ -121,6 +122,32 @@ def _expression(self, species_contam): return sorted(k for k, v in species_contam['contaminant_unique_mapped'].items() if v > 500) +class RequiredYield(Calculation): + quantised_yields = cfg['available_yields'] + + def _expression(self, genome_size): + return { + required_coverage: self._get_yield(genome_size, required_coverage) + for required_coverage in cfg['available_coverages'] + } + + def _get_yield(self, genome_size, required_coverage): + exact_yield = genome_size * required_coverage + for y in sorted(self.quantised_yields): + if y >= exact_yield: + return y + + raise ValueError( + 'Could not resolve a required yield for %sX coverage and genome size %s' % (required_coverage, genome_size) + ) + + +class RequiredYieldQ30(RequiredYield): + def _get_yield(self, genome_size, required_coverage): + required_yield = super()._get_yield(genome_size, required_coverage) + return self.quantised_yields[required_yield] + + class MostRecent(Calculation): # TODO: Should replace server_side.MostRecent def __init__(self, *args, date_field='_created'): self.date_field = date_field @@ -386,6 +413,15 @@ def superelements(self): return {endpoint: {id_field: self.data['dataset_name']}} +class Species(DataRelation): + endpoint = 'species' + id_field = 'name' + aggregated_fields = { + 'required_yield': RequiredYield('approximate_genome_size'), + 'required_yield_q30': RequiredYieldQ30('approximate_genome_size') + } + + data_relations = { cls.endpoint: cls for cls in (Run, Lane, RunElement, Sample, Project, AnalysisDriverProc) diff --git a/tests/test_rest_api/test_aggregation/test_database_hooks.py b/tests/test_rest_api/test_aggregation/test_database_hooks.py index 56e0f10d..457ea8c9 100644 --- a/tests/test_rest_api/test_aggregation/test_database_hooks.py +++ b/tests/test_rest_api/test_aggregation/test_database_hooks.py @@ -476,3 +476,25 @@ def test_nb_unique_mutable_elements(): e = database_hooks.NbUniqueDicts('things', key='this', filter_func=lambda x: x['that'] > 1) assert e.evaluate({'things': data}) == 2 + + +def test_required_yield(): + e = database_hooks.RequiredYield('approximate_genome_size') + assert e.evaluate({'approximate_genome_size': 2}) == { + 1: 5, + 2: 5, + 5: 10, + 10: 20, + 20: 40 + } + + +def test_yield_for_quoted_coverage(): + e = database_hooks.RequiredYieldQ30('approximate_genome_size') + assert e.evaluate({'approximate_genome_size': 2}) == { + 1: 4, + 2: 4, + 5: 8, + 10: 16, + 20: 32 + } From c8f09603d026c9420ed5c7b322406a8aeab04a0d Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Wed, 24 Oct 2018 16:49:46 +0100 Subject: [PATCH 3/9] Fixing build --- tests/test_rest_api/test_schema.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_rest_api/test_schema.py b/tests/test_rest_api/test_schema.py index 10812caa..54d73309 100644 --- a/tests/test_rest_api/test_schema.py +++ b/tests/test_rest_api/test_schema.py @@ -1,5 +1,6 @@ from egcg_core import constants from config import schema +from rest_api import settings from tests import Helper @@ -26,16 +27,18 @@ def resolve_dict_value_in_constant(schema_provided, islist=False): islist=schema_provided[field]['type'] == 'list' ) - for endpoint in ('runs', 'lanes', 'run_elements', 'unexpected_barcodes', 'projects', 'samples', - 'analysis_driver_procs', 'analysis_driver_stages', 'actions'): + for endpoint in settings.DOMAIN: resolve_dict_value_in_constant(schema[endpoint]) expected_not_in_constant = [ - 'action_id', 'action_info', 'action_type', 'analysis_driver_proc', 'best_matching_samples', 'data_deleted', - 'data_source', 'dataset_name', 'dataset_type', 'date_finished', 'date_started', 'end_date', 'exit_status', - 'files_delivered', 'files_downloaded', 'gender_genotype', 'name', 'pid', 'pipeline_used', 'sample_pipeline', - 'stage_id', 'stage_name', 'stages', 'start_date', 'started_by', 'toolset_type', 'toolset_version', - 'useable_reviewer' + 'action_id', 'action_info', 'action_type', 'analyses_supported', 'analysis_driver_proc', + 'approximate_genome_size', 'assembly_name', 'best_matching_samples', 'chromosome_count', 'comments', + 'data_deleted', 'data_files', 'data_source', 'dataset_name', 'dataset_type', 'date_finished', + 'date_started', 'default_version', 'end_date', 'exit_status', 'fasta', 'files_delivered', + 'files_downloaded', 'gender_genotype', 'genome_used', 'genomes', 'goldenpath', 'name', 'pid', + 'pipeline_used', 'project_whitelist', 'sample_pipeline', 'species', 'stage_id', 'stage_name', 'stages', + 'start_date', 'started_by', 'taxid', 'tools_used', 'toolset_type', 'toolset_version', 'useable_reviewer', + 'variation' ] assert sorted(list_values_not_in_constants) == expected_not_in_constant From d6a377bcd6fffd663def906bb222d7ca53802bcb Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Thu, 25 Oct 2018 16:29:03 +0100 Subject: [PATCH 4/9] Trying datatable columns for required_yield/q30, fixing species aggregation --- etc/column_mappings.yaml | 4 ++++ .../static/stylesheets/reporting_app.css | 2 +- rest_api/aggregation/database_hooks.py | 20 ++++++++++++------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/etc/column_mappings.yaml b/etc/column_mappings.yaml index aea4097b..9f975b95 100644 --- a/etc/column_mappings.yaml +++ b/etc/column_mappings.yaml @@ -117,6 +117,8 @@ column_def: species_name: { data: 'name', title: 'Name' } species_default_genome: { data: 'default_version', title: 'Default genome version' } species_taxid: { data: 'taxid', title: 'Taxid' } + species_yields: { data: 'aggregated.required_yield', title: 'Required yield per coverage' } + species_yield_q30s: { data: 'aggregated.required_yield_q30', title: 'Required yield Q30 per coverage' } genome_assembly: { data: 'assembly_name', title: 'Assembly name' } genome_species: { data: 'species', title: 'Species' } genome_data_source: { data: 'data_source', title: 'Data source' } @@ -370,6 +372,8 @@ species: - species_name - species_default_genome - species_taxid + - species_yields + - species_yield_q30s genomes: diff --git a/reporting_app/static/stylesheets/reporting_app.css b/reporting_app/static/stylesheets/reporting_app.css index 65ef2483..82616f8f 100644 --- a/reporting_app/static/stylesheets/reporting_app.css +++ b/reporting_app/static/stylesheets/reporting_app.css @@ -17,7 +17,6 @@ } .dropdown { - padding: 2px; //display: inline-block; } @@ -36,6 +35,7 @@ .dropdown-content { display: none; + padding: 2px; position: absolute; z-index: 10; background-color: #f9f9f9; diff --git a/rest_api/aggregation/database_hooks.py b/rest_api/aggregation/database_hooks.py index 9fa85e3e..a9edcdea 100644 --- a/rest_api/aggregation/database_hooks.py +++ b/rest_api/aggregation/database_hooks.py @@ -1,4 +1,5 @@ import statistics +from cached_property import cached_property from rest_api.aggregation.server_side.expressions import * from rest_api.aggregation.database_side import db from rest_api import cfg @@ -123,11 +124,14 @@ def _expression(self, species_contam): class RequiredYield(Calculation): - quantised_yields = cfg['available_yields'] + @cached_property + def quantised_yields(self): + return cfg['available_yields'] def _expression(self, genome_size): return { - required_coverage: self._get_yield(genome_size, required_coverage) + # keys have to be strings to be valid BSON + str(required_coverage) + 'X': self._get_yield(genome_size, required_coverage) for required_coverage in cfg['available_coverages'] } @@ -416,15 +420,17 @@ def superelements(self): class Species(DataRelation): endpoint = 'species' id_field = 'name' - aggregated_fields = { - 'required_yield': RequiredYield('approximate_genome_size'), - 'required_yield_q30': RequiredYieldQ30('approximate_genome_size') - } + aggregated_fields = [ + { + 'required_yield': RequiredYield('approximate_genome_size'), + 'required_yield_q30': RequiredYieldQ30('approximate_genome_size') + } + ] data_relations = { cls.endpoint: cls - for cls in (Run, Lane, RunElement, Sample, Project, AnalysisDriverProc) + for cls in (Run, Lane, RunElement, Sample, Project, AnalysisDriverProc, Species) } From d74a1890790f9ae7991486c0a47302ca9c3d9d68 Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Tue, 13 Nov 2018 18:12:24 +0000 Subject: [PATCH 5/9] Using separate datatable for required_yields/q30s, adding custom coverage rendering, calling render_data only when render type is 'display', using changing 'genomes' endpoint to 'species', fixing tests. --- etc/column_mappings.yaml | 24 ++++----- etc/schema.yaml | 2 +- reporting_app/__init__.py | 31 ++++++++++-- reporting_app/static/datatable.js | 49 ++++++++++++++++++- .../test_aggregation/test_database_hooks.py | 20 ++++---- 5 files changed, 96 insertions(+), 30 deletions(-) diff --git a/etc/column_mappings.yaml b/etc/column_mappings.yaml index 9f975b95..70328aae 100644 --- a/etc/column_mappings.yaml +++ b/etc/column_mappings.yaml @@ -117,8 +117,6 @@ column_def: species_name: { data: 'name', title: 'Name' } species_default_genome: { data: 'default_version', title: 'Default genome version' } species_taxid: { data: 'taxid', title: 'Taxid' } - species_yields: { data: 'aggregated.required_yield', title: 'Required yield per coverage' } - species_yield_q30s: { data: 'aggregated.required_yield_q30', title: 'Required yield Q30 per coverage' } genome_assembly: { data: 'assembly_name', title: 'Assembly name' } genome_species: { data: 'species', title: 'Species' } genome_data_source: { data: 'data_source', title: 'Data source' } @@ -129,6 +127,10 @@ column_def: genome_projects: { data: 'project_whitelist', title: 'Allowed projects', visible: 'false' } genome_comments: { data: 'comments', title: 'Comments', visible: 'false' } genome_analyses: { data: 'analyses_supported', title: 'Analyses supported' } + yield_x_coverage: { data: { display: 'coverage.disp', sort: 'coverage.sort' }, title: 'Coverage (X)' } + yield_required: { data: 'required_yield', title: 'Required yield' } + yield_required_q30: { data: 'required_yield_q30', title: 'Required yield Q30' } + # Datatables tables definition @@ -358,22 +360,10 @@ sample_run_elements: - useable -active_runs: - - run_id - - instrument_id - - run_status - - created_date - - cst_date - - project_ids - - sample_ids - - species: - species_name - species_default_genome - species_taxid - - species_yields - - species_yield_q30s genomes: @@ -387,3 +377,9 @@ genomes: - genome_tools_used - genome_projects - genome_comments + + +yields: + - yield_x_coverage + - yield_required + - yield_required_q30 diff --git a/etc/schema.yaml b/etc/schema.yaml index 9420df7c..297d5150 100644 --- a/etc/schema.yaml +++ b/etc/schema.yaml @@ -267,7 +267,7 @@ species: genomes: { type: 'list', schema: { type: 'string', data_relation: { resource: 'genomes', field: 'assembly_name', embeddable: True } } } default_version: { type: 'string' } taxid: { type: 'string' } - approximate_genome_size: { type: 'integer' } + approximate_genome_size: { type: 'float' } # genome size in Mb genomes: diff --git a/reporting_app/__init__.py b/reporting_app/__init__.py index 81b2f3eb..154339ca 100644 --- a/reporting_app/__init__.py +++ b/reporting_app/__init__.py @@ -455,12 +455,12 @@ def project_status_reports(prj_status): ) -@app.route('/genomes') +@app.route('/species') @flask_login.login_required -def genome_page(): +def all_species(): return render_template( 'untabbed_datatables.html', - 'Genomes', + 'Species', tables=[ util.datatable_cfg( 'Available species', @@ -474,3 +474,28 @@ def genome_page(): ) ] ) + + +@app.route('/species/') +def species_page(species): + return render_template( + 'untabbed_datatables.html', + species, + tables=[ + util.datatable_cfg( + 'Summary', + 'species', + util.construct_url('species', all_pages=True, where={'name': species}), + minimal=True + ), + util.datatable_cfg( + 'Yield requirements', + 'yields', + ajax_call={ + 'func_name': 'required_yields', + 'api_url': util.construct_url('species', all_pages=True) + }, + minimal=True + ) + ] + ) diff --git a/reporting_app/static/datatable.js b/reporting_app/static/datatable.js index 18d74610..c0b57fd6 100644 --- a/reporting_app/static/datatable.js +++ b/reporting_app/static/datatable.js @@ -60,7 +60,7 @@ var merge_on_keep_first = function (list_of_array, key) { var _merge_multi_sources = function(dt_config, merge_func){ return function(data, callback, settings){ var calls = dt_config.ajax_call.api_urls.map( function(api_url){ - return $.ajax({ + return $.ajax({ url: api_url, headers: {'Authorization': dt_config.token }, dataType: 'json', @@ -92,6 +92,47 @@ var merge_multi_sources_keep_first = function(dt_config){ return _merge_multi_sources(dt_config, merge_on_keep_first); } + +var required_yields = function(dt_config) { + return function(data, callback, settings) { + var response; + $.ajax( + { + url: dt_config.ajax_call.api_url, + headers: {'Authorization': dt_config.token}, + dataType: 'json', + async: false, + success: function(result) { response = result; } + } + ); + var d = response.data; + + if (d.length > 1) { + console.warn('data is not of length 1'); + } + + var aggregated_data = d[0]['aggregated']; + var result = []; + + for (k in aggregated_data['required_yield']) { + result.push( + { + 'coverage': {'order': k.slice(0, -1), 'disp': k}, + 'required_yield': aggregated_data['required_yield'][k], + 'required_yield_q30': aggregated_data['required_yield_q30'][k] + } + ) + }; + + callback({ + recordsTotal: result.length, + recordsFiltered: result.length, + data: result + }); + } +} + + var test_exist = function(variable){ if ( variable instanceof Array ) { variable = variable.filter(function(n){ return n != null }); @@ -277,7 +318,11 @@ var configure_dt = function(dt_config) { 'data': c.data, 'name': c.data, 'render': function(data, type, row, meta) { - return render_data(data, type, row, meta, c.fmt) + if (type == 'display') { + return render_data(data, type, row, meta, c.fmt); + } else { + return data; + } }, 'orderable': !c.orderable || String(c.orderable).toLowerCase() == 'true', 'visible': !c.visible || String(c.visible).toLowerCase() == 'true', diff --git a/tests/test_rest_api/test_aggregation/test_database_hooks.py b/tests/test_rest_api/test_aggregation/test_database_hooks.py index 457ea8c9..9e421f82 100644 --- a/tests/test_rest_api/test_aggregation/test_database_hooks.py +++ b/tests/test_rest_api/test_aggregation/test_database_hooks.py @@ -481,20 +481,20 @@ def test_nb_unique_mutable_elements(): def test_required_yield(): e = database_hooks.RequiredYield('approximate_genome_size') assert e.evaluate({'approximate_genome_size': 2}) == { - 1: 5, - 2: 5, - 5: 10, - 10: 20, - 20: 40 + '1X': 5, + '2X': 5, + '5X': 10, + '10X': 20, + '20X': 40 } def test_yield_for_quoted_coverage(): e = database_hooks.RequiredYieldQ30('approximate_genome_size') assert e.evaluate({'approximate_genome_size': 2}) == { - 1: 4, - 2: 4, - 5: 8, - 10: 16, - 20: 32 + '1X': 4, + '2X': 4, + '5X': 8, + '10X': 16, + '20X': 32 } From 7c38cf75e868c5e43949f8bd4a9faa09eed5c5b7 Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Thu, 15 Nov 2018 12:14:06 +0000 Subject: [PATCH 6/9] Fixing header link to species page, adding links to species summary table, replacing ' ' with '+' in link formatting, fixing coverage column sorting, adding genomes.data_added. --- etc/column_mappings.yaml | 4 +-- etc/schema.yaml | 1 + reporting_app/static/column_formatter.js | 2 +- reporting_app/templates/base.html | 2 +- tests/js/test_column_formatter.js | 4 +-- tests/js/test_datatable.js | 40 +++++++++++++++++++++++- tests/test_reporting_app.py | 2 +- tests/test_rest_api/test_schema.py | 2 +- 8 files changed, 48 insertions(+), 9 deletions(-) diff --git a/etc/column_mappings.yaml b/etc/column_mappings.yaml index 70328aae..830547ca 100644 --- a/etc/column_mappings.yaml +++ b/etc/column_mappings.yaml @@ -114,7 +114,7 @@ column_def: trim_r2: { data: 'aggregated.trim_r2', title: 'Read 2 Max length', visible: 'false'} tiles_filtered: { data: 'aggregated.tiles_filtered', title: 'Tile removed', visible: 'false'} family_id: { data: 'Family ID', title: 'Family ID (SGP)', visible: 'false'} - species_name: { data: 'name', title: 'Name' } + species_name: { data: 'name', title: 'Name', fmt: { link: '/species/' } } species_default_genome: { data: 'default_version', title: 'Default genome version' } species_taxid: { data: 'taxid', title: 'Taxid' } genome_assembly: { data: 'assembly_name', title: 'Assembly name' } @@ -127,7 +127,7 @@ column_def: genome_projects: { data: 'project_whitelist', title: 'Allowed projects', visible: 'false' } genome_comments: { data: 'comments', title: 'Comments', visible: 'false' } genome_analyses: { data: 'analyses_supported', title: 'Analyses supported' } - yield_x_coverage: { data: { display: 'coverage.disp', sort: 'coverage.sort' }, title: 'Coverage (X)' } + yield_x_coverage: { data: { display: 'coverage.disp', _: 'coverage.order' }, title: 'Coverage (X)' } yield_required: { data: 'required_yield', title: 'Required yield' } yield_required_q30: { data: 'required_yield_q30', title: 'Required yield Q30' } diff --git a/etc/schema.yaml b/etc/schema.yaml index 297d5150..b88f34e3 100644 --- a/etc/schema.yaml +++ b/etc/schema.yaml @@ -276,6 +276,7 @@ genomes: data_files: { type: 'dict', schema: { fasta: { type: 'string' }, variation: { type: 'string' } } } data_source: { type: 'string' } tools_used: { type: 'dict' } + date_added: { type: 'datetime' } chromosome_count: { type: 'integer' } genome_size: { type: 'integer' } goldenpath: { type: 'integer' } diff --git a/reporting_app/static/column_formatter.js b/reporting_app/static/column_formatter.js index 90250922..9022855b 100644 --- a/reporting_app/static/column_formatter.js +++ b/reporting_app/static/column_formatter.js @@ -57,7 +57,7 @@ function string_formatter(cell_data, fmt, row){ } if (fmt['link']) { - _formatted_data = '' + data + ''; + _formatted_data = '' + data + ''; } var min, max; diff --git a/reporting_app/templates/base.html b/reporting_app/templates/base.html index dfebc0eb..ed347dd3 100644 --- a/reporting_app/templates/base.html +++ b/reporting_app/templates/base.html @@ -77,7 +77,7 @@
  • Project Status - All
    • -
  • Genomes
    • +
  • Species
  • Charts
  • {{ current_user.username }} diff --git a/tests/js/test_column_formatter.js b/tests/js/test_column_formatter.js index 4ef35284..91646b88 100644 --- a/tests/js/test_column_formatter.js +++ b/tests/js/test_column_formatter.js @@ -83,8 +83,8 @@ QUnit.test('string_formatter', function(assert) { ); assert.equal( - string_formatter(['1', '2'], {'link': '/a_local_page/'}, row), - '
    1,2
    ' + string_formatter(['a_page', 'another page'], {'link': '/to/'}, row), + '
    a_page,another page
    ' ); }); diff --git a/tests/js/test_datatable.js b/tests/js/test_datatable.js index 98c9f549..bdbc60a9 100644 --- a/tests/js/test_datatable.js +++ b/tests/js/test_datatable.js @@ -117,9 +117,47 @@ QUnit.test('get_run_review', function(assert) { QUnit.test('configure_buttons', function(assert) { assert.deepEqual( - configure_buttons({'buttons':['colvis']}), + configure_buttons({'buttons': ['colvis']}), [{extend: 'colvis', text: '', titleAttr: 'Filter Columns'}] ); }); + +QUnit.test('required_yields', function(assert) { + var test_data = {}; + + // patching ajax + var original_ajax = $.ajax; + var fake_ajax = function(config) { + config.success( + { + 'data': [{'aggregated': { + 'required_yield': {'15X': 10, '30X': 20}, + 'required_yield_q30': {'15X': 8, '30X': 16} + }}] + } + ); + }; + $.ajax = fake_ajax; + // patching complete + + var captured_result; + var fake_callback = function(cfg) { + captured_result = cfg; + }; + + var _required_yields = required_yields({'ajax_call': {'api_url': 'a_url'}, 'token': 'a_token'}); + _required_yields(undefined, fake_callback, undefined); + + assert.deepEqual( + captured_result.data, + [ + {'coverage': {'order': '15', 'disp': '15X'}, 'required_yield': 10, 'required_yield_q30': 8}, + {'coverage': {'order': '30', 'disp': '30X'}, 'required_yield': 20, 'required_yield_q30': 16} + ] + ) + + $.ajax = original_ajax; // end patch +}); + // TODO: test sum_row_per_column diff --git a/tests/test_reporting_app.py b/tests/test_reporting_app.py index f373b22f..a6773e1e 100644 --- a/tests/test_reporting_app.py +++ b/tests/test_reporting_app.py @@ -332,4 +332,4 @@ def test_project_status_report(self): self._test_render_template('/project_status/') def test_genome_page(self): - self._test_render_template('/genomes') + self._test_render_template('/species') diff --git a/tests/test_rest_api/test_schema.py b/tests/test_rest_api/test_schema.py index 54d73309..cab90b12 100644 --- a/tests/test_rest_api/test_schema.py +++ b/tests/test_rest_api/test_schema.py @@ -33,7 +33,7 @@ def resolve_dict_value_in_constant(schema_provided, islist=False): expected_not_in_constant = [ 'action_id', 'action_info', 'action_type', 'analyses_supported', 'analysis_driver_proc', 'approximate_genome_size', 'assembly_name', 'best_matching_samples', 'chromosome_count', 'comments', - 'data_deleted', 'data_files', 'data_source', 'dataset_name', 'dataset_type', 'date_finished', + 'data_deleted', 'data_files', 'data_source', 'dataset_name', 'dataset_type', 'date_added', 'date_finished', 'date_started', 'default_version', 'end_date', 'exit_status', 'fasta', 'files_delivered', 'files_downloaded', 'gender_genotype', 'genome_used', 'genomes', 'goldenpath', 'name', 'pid', 'pipeline_used', 'project_whitelist', 'sample_pipeline', 'species', 'stage_id', 'stage_name', 'stages', From 14051645de5cb2f8b714083c6eda7ef92c11633e Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Thu, 15 Nov 2018 12:48:00 +0000 Subject: [PATCH 7/9] RequiredYield._get_yield: removing ValueError, adding docstrings to clarify Mb/Gb. Using Mb in required yield tests. --- rest_api/aggregation/database_hooks.py | 17 ++++++++++++----- .../test_aggregation/test_database_hooks.py | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/rest_api/aggregation/database_hooks.py b/rest_api/aggregation/database_hooks.py index a9edcdea..a5c38e68 100644 --- a/rest_api/aggregation/database_hooks.py +++ b/rest_api/aggregation/database_hooks.py @@ -136,18 +136,25 @@ def _expression(self, genome_size): } def _get_yield(self, genome_size, required_coverage): - exact_yield = genome_size * required_coverage + """ + Obtain a required yield, given a particular genome and required X coverage, using the keys of the + 'available_yields' config for candidate yield values. + :param float genome_size: Genome size in Mb + :param int required_coverage: X coverage + :return: required yield in Gb + """ + exact_yield = genome_size / 1000 * required_coverage for y in sorted(self.quantised_yields): if y >= exact_yield: return y - raise ValueError( - 'Could not resolve a required yield for %sX coverage and genome size %s' % (required_coverage, genome_size) - ) - class RequiredYieldQ30(RequiredYield): def _get_yield(self, genome_size, required_coverage): + """ + Obtain a required yield as per RequiredYield._get_yield, and use the 'available_yields' config to translate it + into a required yield q30. + """ required_yield = super()._get_yield(genome_size, required_coverage) return self.quantised_yields[required_yield] diff --git a/tests/test_rest_api/test_aggregation/test_database_hooks.py b/tests/test_rest_api/test_aggregation/test_database_hooks.py index 9e421f82..742beb07 100644 --- a/tests/test_rest_api/test_aggregation/test_database_hooks.py +++ b/tests/test_rest_api/test_aggregation/test_database_hooks.py @@ -480,7 +480,7 @@ def test_nb_unique_mutable_elements(): def test_required_yield(): e = database_hooks.RequiredYield('approximate_genome_size') - assert e.evaluate({'approximate_genome_size': 2}) == { + assert e.evaluate({'approximate_genome_size': 1990.6}) == { '1X': 5, '2X': 5, '5X': 10, @@ -491,7 +491,7 @@ def test_required_yield(): def test_yield_for_quoted_coverage(): e = database_hooks.RequiredYieldQ30('approximate_genome_size') - assert e.evaluate({'approximate_genome_size': 2}) == { + assert e.evaluate({'approximate_genome_size': 1990.6}) == { '1X': 4, '2X': 4, '5X': 8, From 191e01fcc04bbfdb96aec71a234ebd51834c7e90 Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Mon, 19 Nov 2018 15:24:46 +0000 Subject: [PATCH 8/9] Adding example yields to docker/reporting.yaml, adding genome size and date added to species tables and sorting by X coverage, documenting/reindenting/simplifying string_formatter, fixing datatable queries in species endpoints, fixing default_sort_order for orthogonal data. --- config.py | 6 +++ docker/reporting.yaml | 9 ++++ etc/column_mappings.yaml | 14 +++++-- reporting_app/__init__.py | 10 +++-- reporting_app/static/column_formatter.js | 52 ++++++++++++------------ reporting_app/util.py | 5 ++- tests/test_reporting_app.py | 8 ++-- 7 files changed, 66 insertions(+), 38 deletions(-) diff --git a/config.py b/config.py index aeee401f..f664106c 100644 --- a/config.py +++ b/config.py @@ -24,11 +24,13 @@ def __init__(self, *cfg_search_path): # self.content[key] needs to be a list of strings or dicts for i in range(len(self.content[key])): if isinstance(self.content[key][i], str): + column_name = self.content[key][i] if self.content[key][i] in self.column_def: self.content[key][i] = self.column_def[self.content[key][i]] else: raise ConfigError('No column definition for %s' % self.content[key][i]) elif isinstance(self.content[key][i], dict): + column_name = self.content[key][i]['column_def'] if self.content[key][i]['column_def'] in self.column_def: # take a copy of the column def and update it with the specific info tmp = copy.copy(self.column_def[self.content[key][i]['column_def']]) @@ -37,6 +39,10 @@ def __init__(self, *cfg_search_path): else: # leave the definition as it is pass + else: + raise ConfigError('Invalid column definition %s' % self.content[key][i]) + + self.content[key][i]['name'] = column_name class ProjectStatusConfig(Configuration): diff --git a/docker/reporting.yaml b/docker/reporting.yaml index d048908c..57028c51 100644 --- a/docker/reporting.yaml +++ b/docker/reporting.yaml @@ -18,6 +18,15 @@ rest_app: url: a_baseurl db: a_db_name + available_coverages: [10, 15, 30, 60, 90, 120] + available_yields: + 40: 35 + 60: 52.5 + 120: 105 + 240: 210 + 360: 315 + 480: 420 + reporting_app: # only need this in the image for the user_db key: 'a_test_key' diff --git a/etc/column_mappings.yaml b/etc/column_mappings.yaml index 830547ca..9b93259c 100644 --- a/etc/column_mappings.yaml +++ b/etc/column_mappings.yaml @@ -1,13 +1,15 @@ # Datatables column definitions for the reporting app # Each column definition has a key and a dict with the following options: -# - data: the field in the javascript object where the data is found (support dot notation) +# - data: the field in the javascript object where the data is found (supports dot notation) +# * this can be an object for rendering/sorting/searching data in different ways - see orthogonal data and +# columns.data in the Datatable docs # - title: header of that column in the datatable # - visible: specify if the column will be visible by default (default is true if not set) # - class_name: space separated list of css class for the column # - fmt: reference and paramters to the cell formatter (see column_formatter.js for detail) # * type: formatter type (int, float, percentage, ratio_percentage, date, datetime) -# * min: minimum bellow which the value is highlighted -# * max: maximum bellow which the value is highlighted +# * min: minimum below which the value is highlighted +# * max: maximum above which the value is highlighted # * link: relative link formatted as baseurl//data # * link_format_function: javascript function name formatting the link's text # * name: javascript function name that will modify the data before the formating @@ -117,17 +119,19 @@ column_def: species_name: { data: 'name', title: 'Name', fmt: { link: '/species/' } } species_default_genome: { data: 'default_version', title: 'Default genome version' } species_taxid: { data: 'taxid', title: 'Taxid' } + species_genome_size: { data: 'approximate_genome_size', title: 'Genome size (Mb)' } genome_assembly: { data: 'assembly_name', title: 'Assembly name' } genome_species: { data: 'species', title: 'Species' } genome_data_source: { data: 'data_source', title: 'Data source' } genome_tools_used: { data: 'tools_used', title: 'Tools used', visible: 'false' } + genome_date_added: { data: 'date_added', title: 'Date added' } genome_chromosome_count: { data: 'chromosome_count', title: 'Chromosomes' } genome_size: { data: 'genome_size', title: 'Genome size' } genome_goldenpath: { data: 'goldenpath', title: 'GoldenPath' } genome_projects: { data: 'project_whitelist', title: 'Allowed projects', visible: 'false' } genome_comments: { data: 'comments', title: 'Comments', visible: 'false' } genome_analyses: { data: 'analyses_supported', title: 'Analyses supported' } - yield_x_coverage: { data: { display: 'coverage.disp', _: 'coverage.order' }, title: 'Coverage (X)' } + yield_x_coverage: { data: { display: 'coverage.disp', _: 'coverage.order' }, title: 'Coverage' } # e.g, render as '30X' but sort, search, etc. as '30' yield_required: { data: 'required_yield', title: 'Required yield' } yield_required_q30: { data: 'required_yield_q30', title: 'Required yield Q30' } @@ -364,6 +368,7 @@ species: - species_name - species_default_genome - species_taxid + - species_genome_size genomes: @@ -376,6 +381,7 @@ genomes: - genome_analyses - genome_tools_used - genome_projects + - genome_date_added - genome_comments diff --git a/reporting_app/__init__.py b/reporting_app/__init__.py index 154339ca..a5bd613b 100644 --- a/reporting_app/__init__.py +++ b/reporting_app/__init__.py @@ -465,18 +465,19 @@ def all_species(): util.datatable_cfg( 'Available species', 'species', - util.construct_url('species', all_pages=True) + util.construct_url('species', max_results=1000) ), util.datatable_cfg( 'Installed genomes', 'genomes', - util.construct_url('genomes', all_pages=True) + util.construct_url('genomes', max_results=10000) ) ] ) @app.route('/species/') +@flask_login.login_required def species_page(species): return render_template( 'untabbed_datatables.html', @@ -485,7 +486,7 @@ def species_page(species): util.datatable_cfg( 'Summary', 'species', - util.construct_url('species', all_pages=True, where={'name': species}), + util.construct_url('species', where={'name': species}), minimal=True ), util.datatable_cfg( @@ -493,8 +494,9 @@ def species_page(species): 'yields', ajax_call={ 'func_name': 'required_yields', - 'api_url': util.construct_url('species', all_pages=True) + 'api_url': util.construct_url('species', where={'name': species}) }, + default_sort_col='yield_x_coverage', minimal=True ) ] diff --git a/reporting_app/static/column_formatter.js b/reporting_app/static/column_formatter.js index 9022855b..046ee5f1 100644 --- a/reporting_app/static/column_formatter.js +++ b/reporting_app/static/column_formatter.js @@ -18,24 +18,22 @@ function render_data(data, type, row, meta, fmt) { function string_formatter(cell_data, fmt, row){ + // cast the cell data to a list, whether it's a single value, an object or already a list + // this allows subsequent logic to safely assume it's handling a list if (cell_data instanceof Array) { cell_data.sort(); } else if (cell_data instanceof Object) { + // convert, e.g, {'this': 0, 'that': 1, 'other': 2} to ['this: 0', 'that: 1', 'other: 2'] var _cell_data = []; for (k in cell_data) { _cell_data.push(k + ': ' + cell_data[k]); } cell_data = _cell_data; } else { - cell_data = [cell_data]; + cell_data = [cell_data]; // cast a single value to a list of length 1 } - var use_dropdown = false; - if (cell_data.length > 1) { - use_dropdown = true; - } var formatted_data = []; - for (var i=0, tot=cell_data.length; i' + data + ''; - } + if (fmt['link']) { // convert the link to an html , replacing ' ' with '+' + _formatted_data = '' + data + ''; + } - var min, max; - if (fmt['min']) { - min = resolve_min_max_value(row, fmt['min']) - } - if (fmt['max']) { - max = resolve_min_max_value(row, fmt['max']) - } - if (min && data < min) { - _formatted_data = '
    ' + _formatted_data + '
    '; - } else if (max && !isNaN(max) && data > max) { - _formatted_data = '
    ' + _formatted_data + '
    '; - } else if (max && data > max) { - _formatted_data = '
    ' + _formatted_data + '
    '; - } + var min, max; + if (fmt['min']) { + min = resolve_min_max_value(row, fmt['min']) + } + if (fmt['max']) { + max = resolve_min_max_value(row, fmt['max']) + } + if (min && data < min) { + _formatted_data = '
    ' + _formatted_data + '
    '; + } else if (max && !isNaN(max) && data > max) { + _formatted_data = '
    ' + _formatted_data + '
    '; + } else if (max && data > max) { + _formatted_data = '
    ' + _formatted_data + '
    '; + } - formatted_data.push(_formatted_data); + formatted_data.push(_formatted_data); } - if (use_dropdown) { + // if the list is longer than 1, then it should be rendered as a dropdown + if (formatted_data.length > 1) { + // build a
    text or link
    var dropdown = document.createElement('div'); dropdown.className = 'dropdown'; @@ -104,6 +104,8 @@ function string_formatter(cell_data, fmt, row){ dropdown.appendChild(dropbtn); dropdown.appendChild(dropdown_content); formatted_data = dropdown.outerHTML; + } else { + formatted_data = formatted_data[0]; } return '
    ' + formatted_data + '
    '; diff --git a/reporting_app/util.py b/reporting_app/util.py index d6e8bd79..2fda2ce9 100644 --- a/reporting_app/util.py +++ b/reporting_app/util.py @@ -9,7 +9,10 @@ def _format_order(col_name, cols): direction = 'desc' if col_name.startswith('-') else 'asc' - return [[c['data'] for c in cols].index(col_name.lstrip('-')), direction] + return [ + [c['name'] for c in cols].index(col_name.lstrip('-')), + direction + ] def construct_url(endpoint, **query_args): diff --git a/tests/test_reporting_app.py b/tests/test_reporting_app.py index a6773e1e..659016f2 100644 --- a/tests/test_reporting_app.py +++ b/tests/test_reporting_app.py @@ -76,10 +76,10 @@ def tearDownClass(cls): def test_format_order(self): cols = ( - {'data': 'this', 'title': 'This'}, - {'data': 'that', 'title': 'That'}, - {'data': 'other', 'title': 'Other'}, - {'data': 'another', 'title': 'Another'} + {'name': 'this', 'title': 'This'}, + {'name': 'that', 'title': 'That'}, + {'name': 'other', 'title': 'Other'}, + {'name': 'another', 'title': 'Another'} ) assert reporting_app.util._format_order('this', cols) == [0, 'asc'] assert reporting_app.util._format_order('-other', cols) == [2, 'desc'] From 65877c0595c834a8c65ec06d38ba332e4f13b39e Mon Sep 17 00:00:00 2001 From: Murray Wham Date: Tue, 20 Nov 2018 12:19:23 +0000 Subject: [PATCH 9/9] Adding supported genomes to species page, returning None from RequiredYieldQ30._get_yield if no yield found --- etc/example_reporting.yaml | 2 +- reporting_app/__init__.py | 6 ++++++ rest_api/aggregation/database_hooks.py | 3 ++- tests/test_reporting_app.py | 1 + .../test_aggregation/test_database_hooks.py | 18 ++++++++++-------- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/etc/example_reporting.yaml b/etc/example_reporting.yaml index e2876a6e..bcdd65e0 100644 --- a/etc/example_reporting.yaml +++ b/etc/example_reporting.yaml @@ -51,7 +51,7 @@ rest_app: username: user password: pass - available_coverages: [1, 2, 5, 10, 20] + available_coverages: [1, 2, 4, 5, 10, 20] available_yields: # keys are required yields, and values are associated required yield q30s 5: 4 10: 8 diff --git a/reporting_app/__init__.py b/reporting_app/__init__.py index a5bd613b..2d1f01d4 100644 --- a/reporting_app/__init__.py +++ b/reporting_app/__init__.py @@ -489,6 +489,12 @@ def species_page(species): util.construct_url('species', where={'name': species}), minimal=True ), + util.datatable_cfg( + 'Supported genomes', + 'genomes', + util.construct_url('genomes', where={'species': species}, max_results=1000), + minimal=True + ), util.datatable_cfg( 'Yield requirements', 'yields', diff --git a/rest_api/aggregation/database_hooks.py b/rest_api/aggregation/database_hooks.py index a5c38e68..e01ba7b0 100644 --- a/rest_api/aggregation/database_hooks.py +++ b/rest_api/aggregation/database_hooks.py @@ -156,7 +156,8 @@ def _get_yield(self, genome_size, required_coverage): into a required yield q30. """ required_yield = super()._get_yield(genome_size, required_coverage) - return self.quantised_yields[required_yield] + if required_yield: + return self.quantised_yields[required_yield] class MostRecent(Calculation): # TODO: Should replace server_side.MostRecent diff --git a/tests/test_reporting_app.py b/tests/test_reporting_app.py index 659016f2..61c4a8ae 100644 --- a/tests/test_reporting_app.py +++ b/tests/test_reporting_app.py @@ -333,3 +333,4 @@ def test_project_status_report(self): def test_genome_page(self): self._test_render_template('/species') + self._test_render_template('/species/a_species') diff --git a/tests/test_rest_api/test_aggregation/test_database_hooks.py b/tests/test_rest_api/test_aggregation/test_database_hooks.py index 742beb07..21afc022 100644 --- a/tests/test_rest_api/test_aggregation/test_database_hooks.py +++ b/tests/test_rest_api/test_aggregation/test_database_hooks.py @@ -480,21 +480,23 @@ def test_nb_unique_mutable_elements(): def test_required_yield(): e = database_hooks.RequiredYield('approximate_genome_size') - assert e.evaluate({'approximate_genome_size': 1990.6}) == { + assert e.evaluate({'approximate_genome_size': 2100.6}) == { '1X': 5, '2X': 5, - '5X': 10, - '10X': 20, - '20X': 40 + '4X': 10, + '5X': 20, + '10X': 40, + '20X': None } def test_yield_for_quoted_coverage(): e = database_hooks.RequiredYieldQ30('approximate_genome_size') - assert e.evaluate({'approximate_genome_size': 1990.6}) == { + assert e.evaluate({'approximate_genome_size': 2100.6}) == { '1X': 4, '2X': 4, - '5X': 8, - '10X': 16, - '20X': 32 + '4X': 8, + '5X': 16, + '10X': 32, + '20X': None }