From dc3b3b3adaa3ec411ec5ef9a71fab47e185cef2c Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Fri, 1 Sep 2023 16:22:21 -0700 Subject: [PATCH] Web Dashboard: added a page for displaying query progress at Czar Added a link (an additional column on the active queries table) to the newely added Query progression plot. Fixed minor layout bugs on the active and past query tables. Loading the Highcharts.js library on demand. --- src/www/dashboard.html | 2 + src/www/qserv/css/QservCzarQueryProgress.css | 60 ++++ src/www/qserv/js/QservCzarQueryProgress.js | 301 +++++++++++++++++++ src/www/qserv/js/QservMonitoringDashboard.js | 16 +- src/www/qserv/js/StatusActiveQueries.js | 14 +- src/www/qserv/js/StatusPastQueries.js | 4 +- 6 files changed, 388 insertions(+), 9 deletions(-) create mode 100644 src/www/qserv/css/QservCzarQueryProgress.css create mode 100644 src/www/qserv/js/QservCzarQueryProgress.js diff --git a/src/www/dashboard.html b/src/www/dashboard.html index 154f4288d..437f2690d 100644 --- a/src/www/dashboard.html +++ b/src/www/dashboard.html @@ -4,7 +4,9 @@ Qserv monitoring dashboard + + diff --git a/src/www/qserv/css/QservCzarQueryProgress.css b/src/www/qserv/css/QservCzarQueryProgress.css new file mode 100644 index 000000000..f182539c6 --- /dev/null +++ b/src/www/qserv/css/QservCzarQueryProgress.css @@ -0,0 +1,60 @@ +#fwk-qserv-czar-query-prog-controls label { + font-weight: bold; +} +table#fwk-qserv-czar-query-prog-status { + margin:0; +} +table#fwk-qserv-czar-query-prog-status caption { + caption-side: top; + text-align: right; + padding-top: 0; +} +table#fwk-qserv-czar-query-prog-status caption.updating { + background-color: #ffeeba; +} + +#fwk-qserv-czar-query-prog-status-queries { + height: 712px; +} +.highcharts-figure, +.highcharts-data-table table { + min-width: 310px; + max-width: 800px; + margin: 1em auto; +} + +.highcharts-data-table table { + font-family: Verdana, sans-serif; + border-collapse: collapse; + border: 1px solid #ebebeb; + margin: 10px auto; + text-align: center; + width: 100%; + max-width: 500px; +} + +.highcharts-data-table caption { + padding: 1em 0; + font-size: 1.2em; + color: #555; +} + +.highcharts-data-table th { + font-weight: 600; + padding: 0.5em; +} + +.highcharts-data-table td, +.highcharts-data-table th, +.highcharts-data-table caption { + padding: 0.5em; +} + +.highcharts-data-table thead tr, +.highcharts-data-table tr:nth-child(even) { + background: #f8f8f8; +} + +.highcharts-data-table tr:hover { + background: #f1f7ff; +} diff --git a/src/www/qserv/js/QservCzarQueryProgress.js b/src/www/qserv/js/QservCzarQueryProgress.js new file mode 100644 index 000000000..032bdbd41 --- /dev/null +++ b/src/www/qserv/js/QservCzarQueryProgress.js @@ -0,0 +1,301 @@ +define([ + 'webfwk/CSSLoader', + 'webfwk/Fwk', + 'webfwk/FwkApplication', + 'qserv/Common', + 'underscore', + 'highcharts', + 'highcharts/modules/exporting', + 'highcharts/modules/accessibility'], + +function(CSSLoader, + Fwk, + FwkApplication, + Common, + _, + Highcharts) { + + CSSLoader.load('qserv/css/QservCzarQueryProgress.css'); + + class QservCzarQueryProgress extends FwkApplication { + + constructor(name) { + super(name); + this._data = undefined; + this._queries_chart = undefined; + } + fwk_app_on_show() { + console.log('show: ' + this.fwk_app_name); + this.fwk_app_on_update(); + } + fwk_app_on_hide() { + console.log('hide: ' + this.fwk_app_name); + } + fwk_app_on_update() { + if (this.fwk_app_visible) { + this._init(); + if (this._prev_update_sec === undefined) { + this._prev_update_sec = 0; + } + let now_sec = Fwk.now().sec; + if (now_sec - this._prev_update_sec > this._update_interval_sec()) { + this._prev_update_sec = now_sec; + this._init(); + this._load(); + } + } + } + /// Set the identifier and begin loading the query info in the background. + set_query_id(query_id) { + this._init(); + this._set_query_ids([query_id]); // To get the minimally-polulated selector + this._set_query_id(query_id); + this._set_last_seconds(24 * 3600); // Track the known history (if any) of the query + this._load(); + } + _init() { + if (this._initialized === undefined) this._initialized = false; + if (this._initialized) return; + this._initialized = true; + const lastMinutes = [1, 3, 5, 15, 30, 45]; + const lastHours = [1, 2, 4, 8, 12, 16, 20, 24]; + let html = ` +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ ${Common.html_update_ival('update-interval', 10)} +
+
+ + +
+
+
+
+
+
+ + +
Loading...
+
+
+
+
+
+
+
+`; + let cont = this.fwk_app_container.html(html); + cont.find('[data-toggle="tooltip"]').tooltip(); + this._set_last_seconds(900); + cont.find(".form-control-selector").change(() => { + this._load(); + }); + cont.find(".form-control-viewer").change(() => { + if (_.isUndefined(this._data)) this._load(); + else this._display(this._data.queries); + }); + cont.find("button#reset-form").click(() => { + this._set_update_interval_sec(10); + this._set_query_id(0); + this._set_last_seconds(15 * 60); + this._set_vertical_scale('logarithmic'); + this._set_horizontal_scale(''); + this._load(); + }); + } + _form_control(elem_type, id) { + if (this._form_control_obj === undefined) this._form_control_obj = {}; + if (!_.has(this._form_control_obj, id)) { + this._form_control_obj[id] = this.fwk_app_container.find(elem_type + '#' + id); + } + return this._form_control_obj[id]; + } + _update_interval_sec() { return this._form_control('select', 'update-interval').val(); } + _set_update_interval_sec(val) { this._form_control('select', 'update-interval').val(val); } + _query_id() { return this._form_control('select', 'query-id').val(); } + _set_query_id(val) { + this._form_control('select', 'query-id').val(val); + } + _set_query_ids(queries) { + const prev_query = this._query_id(); + let html = ''; + for (let i in queries) { + const query = queries[i]; + const selected = (_.isEmpty(prev_query) && (i === 0)) || + (!_.isEmpty(prev_query) && (prev_query === query)); + html += ` +`; + } + this._form_control('select', 'query-id').html(html); + } + _vertical_scale() { return this._form_control('select', 'vertical-scale').val(); } + _set_vertical_scale(val) { this._form_control('select', 'vertical-scale').val(val); } + _horizontal_scale() { return this._form_control('select', 'horizontal-scale').val(); } + _set_horizontal_scale(val) { this._form_control('select', 'horizontal-scale').val(val); } + _last_seconds() { return this._form_control('select', 'last-seconds').val(); } + _set_last_seconds(val) { this._form_control('select', 'last-seconds').val(val); } + _table(name) { + if (_.isUndefined(this._table_obj)) this._table_obj = {}; + if (!_.has(this._table_obj, name)) { + this._table_obj[name] = this.fwk_app_container.find('table#fwk-qserv-czar-query-prog-' + name); + } + return this._table_obj[name]; + } + _status() { + if (_.isUndefined(this._status_obj)) { + this._status_obj = this._table('status').children('caption'); + } + return this._status_obj; + } + _queries() { + if (_.isUndefined(this._queries_obj)) { + this._queries_obj = this.fwk_app_container.find('canvas#queries'); + } + return this._queries_obj; + } + _load() { + if (this._loading === undefined) this._loading = false; + if (this._loading) return; + this._loading = true; + this._status().addClass('updating'); + Fwk.web_service_GET( + "replication/qserv/master/queries/active/progress", + { version: Common.RestAPIVersion, + query_id: this._query_id(), + last_seconds: this._last_seconds() + }, + (data) => { + if (data.success) { + this._data = data; + this._display(data.queries); + Fwk.setLastUpdate(this._status()); + } else { + console.log('request failed', this.fwk_app_name, data.error); + this._status().html('' + data.error + ''); + } + this._status().removeClass('updating'); + this._loading = false; + }, + (msg) => { + console.log('request failed', this.fwk_app_name, msg); + this._status().html('No Response'); + this._status().removeClass('updating'); + this._loading = false; + } + ); + } + _display(queries) { + const query_ids = _.keys(queries); + query_ids.sort(); + query_ids.reverse(); + this._set_query_ids(query_ids); // Update a collection of queries in the selector. + // Add a small delta to the points to allow seeing zeroes on the log scale, + // in case the one was requested. + const valueDeltaForLogScale = this._vertical_scale() === 'linear' ? 0 : 0.1; + let series = []; + for (let qid in queries) { + let points = []; + let query_data = queries[qid]; + for (let i in query_data) { + const point = query_data[i]; + // +1 hr is needed for correcting timestamp mismatch between UNIX and JS timing + const timestampSec = point[0] / 1000 + 3600; + let x = new Date(0); + x.setSeconds(timestampSec); + points.push([x.getTime(), point[1] + valueDeltaForLogScale]); + } + series.push({ + name: qid, + data: points, + animation: { + enabled: false + } + }); + } + if (!_.isUndefined(this._queries_chart)) { + this._queries_chart.destroy(); + } + this._queries_chart = Highcharts.chart('fwk-qserv-czar-query-prog-status-queries', { + chart: { + type: 'line' + }, + title: { + text: '# Unfinished Jobs' + }, + subtitle: { + text: '< 24 hours' + }, + xAxis: { + type: 'datetime', + title: { + text: 'Time' + }, + // If auto-zoom is not enabled the plot will go all the way through + // the (viewer's) current time on the right. + max: this._horizontal_scale() === 'auto-zoom-in' ? undefined : new Date().setSeconds(0) + }, + yAxis: { + type: this._vertical_scale(), + title: { + text: '# jobs' + } + }, + tooltip: { + headerFormat: '{series.name}
', + pointFormat: '{point.x:%e. %b}: {point.y:.2f} jobs' + }, + time: { + // To ensure the time stamps are displaye din the (viewer's) local timezone. + timezoneOffset: new Date().getTimezoneOffset() + }, + plotOptions: { + series: { + marker: { + fillColor: '#dddddd', + lineWidth: 2, + lineColor: null + } + } + }, + colors: ['#6CF', '#39F', '#06C', '#036', '#000'], + series: series + }); + } + } + return QservCzarQueryProgress; +}); diff --git a/src/www/qserv/js/QservMonitoringDashboard.js b/src/www/qserv/js/QservMonitoringDashboard.js index e0fd22fa6..22318a387 100644 --- a/src/www/qserv/js/QservMonitoringDashboard.js +++ b/src/www/qserv/js/QservMonitoringDashboard.js @@ -5,10 +5,17 @@ require.config({ waitSeconds: 15, urlArgs: "bust="+new Date().getTime(), + packages: [{ + name: 'highcharts', + main: 'highcharts' + }], + paths: { 'jquery': 'https://code.jquery.com/jquery-3.3.1', 'bootstrap': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.bundle', 'underscore': 'https://underscorejs.org/underscore-umd-min', + 'chartjs': 'https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.0/chart.umd.min', + 'highcharts': 'https://code.highcharts.com', 'webfwk': 'webfwk/js', 'qserv': 'qserv/js', 'modules': 'modules/js' @@ -19,13 +26,7 @@ require.config({ }, 'bootstrap': { 'deps': ['jquery','underscore'] - },/* - 'webfwk/*': { - 'deps': ['underscore'] }, - 'qserv/*': { - 'deps': ['underscore'] - },*/ 'underscore': { 'exports': '_' } @@ -44,6 +45,7 @@ require([ 'qserv/StatusWorkers', 'qserv/QservCzarMySQLQueries', 'qserv/QservCzarStatistics', + 'qserv/QservCzarQueryProgress', 'qserv/QservCss', 'qserv/QservMySQLConnections', 'qserv/QservWorkerMySQLQueries', @@ -86,6 +88,7 @@ function(CSSLoader, StatusWorkers, QservCzarMySQLQueries, QservCzarStatistics, + QservCzarQueryProgress, QservCss, QservMySQLConnections, QservWorkerMySQLQueries, @@ -153,6 +156,7 @@ function(CSSLoader, apps: [ new QservCzarMySQLQueries('MySQL Queries'), new QservCzarStatistics('Statistics'), + new QservCzarQueryProgress('Query Progress'), new QservCss('CSS') ] }, diff --git a/src/www/qserv/js/StatusActiveQueries.js b/src/www/qserv/js/StatusActiveQueries.js index 0f4abaa6c..7978d1be4 100644 --- a/src/www/qserv/js/StatusActiveQueries.js +++ b/src/www/qserv/js/StatusActiveQueries.js @@ -108,6 +108,7 @@ function(CSSLoader, QID + Query @@ -185,6 +186,7 @@ function(CSSLoader, const queryToggleTitle = "Click to toggle query formatting."; const queryCopyTitle = "Click to copy the query text to the clipboard."; const queryInspectTitle = "Click to see detailed info (progress, messages, etc.) on the query."; + const queryProgressTitle = "Click to see query progression plot."; const queryStyle = "color:#4d4dff;"; let html = ''; for (let i in data.queries) { @@ -227,9 +229,12 @@ function(CSSLoader, - + + + + 
` + this._query2text(query.queryId, expanded) + `
`; } @@ -257,10 +262,17 @@ function(CSSLoader, Fwk.find("Status", "Query Inspector").set_query_id(queryId); Fwk.show("Status", "Query Inspector"); }; + let displayQueryProgress = function(e) { + let button = $(e.currentTarget); + let queryId = button.parent().parent().attr("id"); + Fwk.find("Czar", "Query Progress").set_query_id(queryId); + Fwk.show("Czar", "Query Progress"); + }; let tbodyQueries = this._table().children('tbody').html(html); tbodyQueries.find("td.query_toggler").click(toggleQueryDisplay); tbodyQueries.find("button.copy-query").click(copyQueryToClipboard); tbodyQueries.find("button.inspect-query").click(displayQuery); + tbodyQueries.find("button.query-progress").click(displayQueryProgress); } /** diff --git a/src/www/qserv/js/StatusPastQueries.js b/src/www/qserv/js/StatusPastQueries.js index 85a937cab..0326a1ddb 100644 --- a/src/www/qserv/js/StatusPastQueries.js +++ b/src/www/qserv/js/StatusPastQueries.js @@ -314,10 +314,10 @@ function(CSSLoader, 
${query.collectedRows}
${query.finalRows}
${query.queryId}
- + - + 
` + this._query2text(query.queryId, expanded) + `