diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index ac8e4f3..1d1cf99 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -46,9 +46,12 @@ jobs: with: name: 'Data' + - name: 'Filter and compress results file.' + run: python utilities/reduce_output_size.py test_output.csv test_output.csv.gz + - name: move data to the dashboard folder run: | - mv test_output.csv website/dashboard + mv test_output.csv.gz website/dashboard - name: Build documentation run: | diff --git a/utilities/reduce_output_size.py b/utilities/reduce_output_size.py new file mode 100644 index 0000000..cbd9eb8 --- /dev/null +++ b/utilities/reduce_output_size.py @@ -0,0 +1,43 @@ +import os +import gzip +import csv +import sys + +def reduce_output_file_size(input_file:str, output_file:str): + """ + Simplify the data generated by the analysis pipeline by retaining only the essential information required for the frontend. + """ + if os.path.exists(input_file): + # Open the input and output files + with open(input_file, 'r') as infile, gzip.open(output_file, 'wt', newline='') as outfile: + reader = csv.DictReader(infile) + + # Drop b_values columns + fieldnames = [field for field in reader.fieldnames if not field.startswith('bval_')] + writer = csv.DictWriter(outfile, fieldnames=fieldnames) + writer.writeheader() + + columns_to_round = ['f', 'Dp', 'D', 'f_fitted', 'Dp_fitted', 'D_fitted'] + + for row in reader: + #Delete columns starting with 'bval_' + for key in list(row.keys()): + if key.startswith('bval_'): + del row[key] + + # Round values in the remaining relevant columns + for column in columns_to_round: + if column in row: + row[column] = round(float(row[column]), 4) + writer.writerow(row) + else: + print(f"File '{input_file}' not found.") + +if __name__ == '__main__': + if len(sys.argv) != 3: + print("Usage: python reduce_output_size.py ") + sys.exit(1) + + input_file = sys.argv[1] + output_file = sys.argv[2] + reduce_output_file_size(input_file, output_file) diff --git a/website/dashboard/index.html b/website/dashboard/index.html index eefdcaa..e7ca265 100644 --- a/website/dashboard/index.html +++ b/website/dashboard/index.html @@ -8,6 +8,7 @@ + diff --git a/website/dashboard/index.js b/website/dashboard/index.js index f0166ae..5e07ab5 100644 --- a/website/dashboard/index.js +++ b/website/dashboard/index.js @@ -205,17 +205,31 @@ document.addEventListener('DOMContentLoaded', function() { showLoading(); - Papa.parse('test_output.csv', { - download: true, - header: true, - complete: results => { - data = results; - hideLoading(); - populateOptions(data); - drawBoxPlot(); - drawRegionBoxPlot(); - + fetch('test_output.csv.gz') + .then(response => { + if (!response.ok) { + throw new Error('Network response was not ok'); } + return response.arrayBuffer(); + }) + .then(buffer => { + // Use pako to decompress the data + var decompressed = pako.inflate(new Uint8Array(buffer), { to: 'string' }); + // Now use Papa Parse to parse the decompressed CSV data + Papa.parse(decompressed, { + header: true, + complete: results => { + console.log(results); + data = results; + hideLoading(); + populateOptions(data); + drawBoxPlot(); + drawRegionBoxPlot(); + } + }); + }) + .catch(error => { + console.error('There has been a problem with your fetch operation:', error); }); function populateOptions(data) {