Skip to content

Commit

Permalink
Merge pull request #201 from EdinburghGenomics/mwham_develop
Browse files Browse the repository at this point in the history 
Reference data storage
  • Loading branch information
Timothee Cezard authored Nov 30, 2018
2 parents 5aadf21 + 65877c0 commit 57435c8
Show file tree
Hide file tree
Showing 19 changed files with 434 additions and 81 deletions.
6 changes: 6 additions & 0 deletions config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@ def __init__(self, *cfg_search_path):
# self.content[key] needs to be a list of strings or dicts
for i in range(len(self.content[key])):
if isinstance(self.content[key][i], str):
column_name = self.content[key][i]
if self.content[key][i] in self.column_def:
self.content[key][i] = self.column_def[self.content[key][i]]
else:
raise ConfigError('No column definition for %s' % self.content[key][i])
elif isinstance(self.content[key][i], dict):
column_name = self.content[key][i]['column_def']
if self.content[key][i]['column_def'] in self.column_def:
# take a copy of the column def and update it with the specific info
tmp = copy.copy(self.column_def[self.content[key][i]['column_def']])
Expand All @@ -37,6 +39,10 @@ def __init__(self, *cfg_search_path):
else:
# leave the definition as it is
pass
else:
raise ConfigError('Invalid column definition %s' % self.content[key][i])

self.content[key][i]['name'] = column_name


class ProjectStatusConfig(Configuration):
Expand Down
9 changes: 9 additions & 0 deletions docker/reporting.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ rest_app:
url: a_baseurl
db: a_db_name

available_coverages: [10, 15, 30, 60, 90, 120]
available_yields:
40: 35
60: 52.5
120: 105
240: 210
360: 315
480: 420


reporting_app: # only need this in the image for the user_db
key: 'a_test_key'
Expand Down
61 changes: 50 additions & 11 deletions etc/column_mappings.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
# Datatables column definitions for the reporting app
# Each column definition has a key and a dict with the following options:
# - data: the field in the javascript object where the data is found (support dot notation)
# - data: the field in the javascript object where the data is found (supports dot notation)
# * this can be an object for rendering/sorting/searching data in different ways - see orthogonal data and
# columns.data in the Datatable docs
# - title: header of that column in the datatable
# - visible: specify if the column will be visible by default (default is true if not set)
# - class_name: space separated list of css class for the column
# - fmt: reference and paramters to the cell formatter (see column_formatter.js for detail)
# * type: formatter type (int, float, percentage, ratio_percentage, date, datetime)
# * min: minimum bellow which the value is highlighted
# * max: maximum bellow which the value is highlighted
# * min: minimum below which the value is highlighted
# * max: maximum above which the value is highlighted
# * link: relative link formatted as baseurl/<link>/data
# * link_format_function: javascript function name formatting the link's text
# * name: javascript function name that will modify the data before the formating
Expand Down Expand Up @@ -114,6 +116,26 @@ column_def:
trim_r2: { data: 'aggregated.trim_r2', title: 'Read 2 Max length', visible: 'false'}
tiles_filtered: { data: 'aggregated.tiles_filtered', title: 'Tile removed', visible: 'false'}
family_id: { data: 'Family ID', title: 'Family ID (SGP)', visible: 'false'}
species_name: { data: 'name', title: 'Name', fmt: { link: '/species/' } }
species_default_genome: { data: 'default_version', title: 'Default genome version' }
species_taxid: { data: 'taxid', title: 'Taxid' }
species_genome_size: { data: 'approximate_genome_size', title: 'Genome size (Mb)' }
genome_assembly: { data: 'assembly_name', title: 'Assembly name' }
genome_species: { data: 'species', title: 'Species' }
genome_data_source: { data: 'data_source', title: 'Data source' }
genome_tools_used: { data: 'tools_used', title: 'Tools used', visible: 'false' }
genome_date_added: { data: 'date_added', title: 'Date added' }
genome_chromosome_count: { data: 'chromosome_count', title: 'Chromosomes' }
genome_size: { data: 'genome_size', title: 'Genome size' }
genome_goldenpath: { data: 'goldenpath', title: 'GoldenPath' }
genome_projects: { data: 'project_whitelist', title: 'Allowed projects', visible: 'false' }
genome_comments: { data: 'comments', title: 'Comments', visible: 'false' }
genome_analyses: { data: 'analyses_supported', title: 'Analyses supported' }
yield_x_coverage: { data: { display: 'coverage.disp', _: 'coverage.order' }, title: 'Coverage' } # e.g, render as '30X' but sort, search, etc. as '30'
yield_required: { data: 'required_yield', title: 'Required yield' }
yield_required_q30: { data: 'required_yield_q30', title: 'Required yield Q30' }



# Datatables tables definition
# the remaining top level entries each define a table with a list of column
Expand Down Expand Up @@ -342,11 +364,28 @@ sample_run_elements:
- useable


active_runs:
- run_id
- instrument_id
- run_status
- created_date
- cst_date
- project_ids
- sample_ids
species:
- species_name
- species_default_genome
- species_taxid
- species_genome_size


genomes:
- genome_assembly
- genome_species
- genome_data_source
- genome_chromosome_count
- genome_size
- genome_goldenpath
- genome_analyses
- genome_tools_used
- genome_projects
- genome_date_added
- genome_comments


yields:
- yield_x_coverage
- yield_required
- yield_required_q30
7 changes: 7 additions & 0 deletions etc/example_reporting.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,10 @@ rest_app:
baseuri: http://clarity.com
username: user
password: pass

available_coverages: [1, 2, 4, 5, 10, 20]
available_yields: # keys are required yields, and values are associated required yield q30s
5: 4
10: 8
20: 16
40: 32
23 changes: 23 additions & 0 deletions etc/schema.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ analysis_driver_procs:
status: { type: 'string', allowed: ['force_ready', 'processing', 'finished', 'failed', 'aborted', 'reprocess', 'deleted', 'resume'] }
stages: { type: 'list', schema: { type: 'string', data_relation: { resource: 'analysis_driver_stages', field: 'stage_id', embeddable: True } } }
data_source: { type: 'list', required: False, schema: { type: 'string' } }
genome_used: { type: 'string' } # TODO: this should be stored in samples when doing #112

analysis_driver_stages:
stage_id: { type: 'string', required: True, unique: True }
Expand All @@ -251,6 +252,7 @@ analysis_driver_stages:
date_finished: { type: 'datetime', nullable: True }
exit_status: { type: 'integer', nullable: True }


actions:
action_id: { type: 'string', required: True, unique: True }
action_type: { type: 'string', required: True }
Expand All @@ -260,3 +262,24 @@ actions:
action_info: { type: 'dict' }


species:
name: { type: 'string', required: True, unique: True }
genomes: { type: 'list', schema: { type: 'string', data_relation: { resource: 'genomes', field: 'assembly_name', embeddable: True } } }
default_version: { type: 'string' }
taxid: { type: 'string' }
approximate_genome_size: { type: 'float' } # genome size in Mb


genomes:
assembly_name: { type: 'string', required: True, unique: True }
species: { type: 'string' }
data_files: { type: 'dict', schema: { fasta: { type: 'string' }, variation: { type: 'string' } } }
data_source: { type: 'string' }
tools_used: { type: 'dict' }
date_added: { type: 'datetime' }
chromosome_count: { type: 'integer' }
genome_size: { type: 'integer' }
goldenpath: { type: 'integer' }
project_whitelist: { type: 'list', schema: { type: 'string' } }
comments: { type: 'string' }
analyses_supported: { type: 'list', schema: { type: 'string', allowed: ['qc', 'variant_calling', 'bcbio'] } }
54 changes: 54 additions & 0 deletions reporting_app/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,3 +453,57 @@ def project_status_reports(prj_status):
table_foot='sum_row_per_column'
)
)


@app.route('/species')
@flask_login.login_required
def all_species():
return render_template(
'untabbed_datatables.html',
'Species',
tables=[
util.datatable_cfg(
'Available species',
'species',
util.construct_url('species', max_results=1000)
),
util.datatable_cfg(
'Installed genomes',
'genomes',
util.construct_url('genomes', max_results=10000)
)
]
)


@app.route('/species/<species>')
@flask_login.login_required
def species_page(species):
return render_template(
'untabbed_datatables.html',
species,
tables=[
util.datatable_cfg(
'Summary',
'species',
util.construct_url('species', where={'name': species}),
minimal=True
),
util.datatable_cfg(
'Supported genomes',
'genomes',
util.construct_url('genomes', where={'species': species}, max_results=1000),
minimal=True
),
util.datatable_cfg(
'Yield requirements',
'yields',
ajax_call={
'func_name': 'required_yields',
'api_url': util.construct_url('species', where={'name': species})
},
default_sort_col='yield_x_coverage',
minimal=True
)
]
)
131 changes: 84 additions & 47 deletions reporting_app/static/column_formatter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ function render_data(data, type, row, meta, fmt) {
return null;
}
if (!fmt) {
return '<div class="dt_cell">' + data + '</div>';
fmt = {};
}
if (fmt['name']) {
data = function_map[fmt['name']](data, fmt)
Expand All @@ -17,61 +17,98 @@ function render_data(data, type, row, meta, fmt) {
}


function string_formatter(data, fmt, row){
var formatted_data = data;

if (fmt['type'] == 'percentage') {
formatted_data = Humanize.toFixed(formatted_data, 1) + '%';
}if (fmt['type'] == 'ratio_percentage') {
formatted_data = Humanize.toFixed(formatted_data * 100, 1) + '%';
} else if (fmt['type'] == 'int') {
formatted_data = Humanize.intComma(formatted_data);
} else if (fmt['type'] == 'float') {
formatted_data = Humanize.formatNumber(formatted_data, 2);
} else if (fmt['type'] == 'date') {
formatted_data = moment(new Date(formatted_data)).format('YYYY-MM-DD');
} else if (fmt['type'] == 'datetime') {
formatted_data = moment(new Date(formatted_data)).format('YYYY-MM-DD HH:mm:ss');
function string_formatter(cell_data, fmt, row){
// cast the cell data to a list, whether it's a single value, an object or already a list
// this allows subsequent logic to safely assume it's handling a list
if (cell_data instanceof Array) {
cell_data.sort();
} else if (cell_data instanceof Object) {
// convert, e.g, {'this': 0, 'that': 1, 'other': 2} to ['this: 0', 'that: 1', 'other: 2']
var _cell_data = [];
for (k in cell_data) {
_cell_data.push(k + ': ' + cell_data[k]);
}
cell_data = _cell_data;
} else {
cell_data = [cell_data]; // cast a single value to a list of length 1
}
if (fmt['link']) {
if (fmt['link_format_function']){
formatted_link = function_map[fmt['link_format_function']](data, fmt);

var formatted_data = [];
for (var i=0, tot=cell_data.length; i<tot; i++) {
var data = cell_data[i];
var _formatted_data;

if (fmt['type'] == 'percentage') {
_formatted_data = Humanize.toFixed(data, 1) + '%';
} else if (fmt['type'] == 'ratio_percentage') {
_formatted_data = Humanize.toFixed(data * 100, 1) + '%';
} else if (fmt['type'] == 'int') {
_formatted_data = Humanize.intComma(data);
} else if (fmt['type'] == 'float') {
_formatted_data = Humanize.formatNumber(data, 2);
} else if (fmt['type'] == 'date') {
_formatted_data = moment(new Date(data)).format('YYYY-MM-DD');
} else if (fmt['type'] == 'datetime') {
_formatted_data = moment(new Date(data)).format('YYYY-MM-DD HH:mm:ss');
} else {
_formatted_data = data;
}
else{
formatted_link = data;

if (fmt['link']) { // convert the link to an html <a/>, replacing ' ' with '+'
_formatted_data = '<a href=' + fmt['link'] + data.replace(/ /g, "+") + '>' + data + '</a>';
}
if (data instanceof Array && data.length > 1 || data != formatted_link) {
data.sort();
formatted_data = '<div class="dropdown"><div class="dropbtn">' + formatted_link + '</div><div class="dropdown-content">';
for (var i=0, tot=data.length; i < tot; i++){
formatted_data = formatted_data.concat('<a href=' + fmt['link'] + data[i] + '>' + data[i] + '</a>');
}
formatted_data = formatted_data.concat('</div></div>')

var min, max;
if (fmt['min']) {
min = resolve_min_max_value(row, fmt['min'])
}
else if (data instanceof Array && data.length == 1){
formatted_data = '<a href=' + fmt['link'] + data[0] + '>' + data[0] + '</a>';
if (fmt['max']) {
max = resolve_min_max_value(row, fmt['max'])
}
else {
formatted_data = '<a href=' + fmt['link'] + data + '>' + data + '</a>';
if (min && data < min) {
_formatted_data = '<div style="color:red">' + _formatted_data + '</div>';
} else if (max && !isNaN(max) && data > max) {
_formatted_data = '<div style="color:red">' + _formatted_data + '</div>';
} else if (max && data > max) {
_formatted_data = '<div style="color:red">' + _formatted_data + '</div>';
}

formatted_data.push(_formatted_data);

}
var min, max;
if (fmt['min']){
min = resolve_min_max_value(row, fmt['min'])
}
if (fmt['max']){
max = resolve_min_max_value(row, fmt['max'])
}
if (min && data < min) {
formatted_data = '<div style="color:red">' + formatted_data + '</div>';
} else if (max && !isNaN(max) && data > max) {
formatted_data = '<div style="color:red">' + formatted_data + '</div>';
} else if (max && data > max) {
formatted_data = '<div style="color:red">' + formatted_data + '</div>';

// if the list is longer than 1, then it should be rendered as a dropdown
if (formatted_data.length > 1) {
// build a <div class="dropdown"><div class="dropbtn">text or link</div></div>
var dropdown = document.createElement('div');
dropdown.className = 'dropdown';

var dropbtn = document.createElement('div');
dropbtn.className = 'dropbtn';
if (fmt['link_format_function']) {
dropbtn.innerHTML = function_map[fmt['link_format_function']](cell_data, fmt);
} else {
dropbtn.innerHTML = cell_data;
}

var dropdown_content = document.createElement('div');
dropdown_content.className = 'dropdown-content';

var div;
for (var i=0, tot=formatted_data.length; i<tot; i++) {
div = document.createElement('div');
div.innerHTML = formatted_data[i];
dropdown_content.appendChild(div);
}

dropdown.appendChild(dropbtn);
dropdown.appendChild(dropdown_content);
formatted_data = dropdown.outerHTML;
} else {
formatted_data = formatted_data[0];
}

formatted_data = '<div class="dt_cell">' + formatted_data + '</div>';
return formatted_data;
return '<div class="dt_cell">' + formatted_data + '</div>';
}


Expand Down
Loading

0 comments on commit 57435c8

Please sign in to comment.