diff --git a/app/controllers/dataController.js b/app/controllers/dataController.js index 3a49496..a906346 100644 --- a/app/controllers/dataController.js +++ b/app/controllers/dataController.js @@ -76,10 +76,10 @@ exports.getUsers = (userId) => { }); }; -exports.getIPs = (userId) => { +exports.getTopIPs = (userId) => { logger.trace(); return new promise((resolve, reject) => { - db.ip.getRecent() + db.ip.getTop() .then(data => { db.user.logAction(userId, 'listIPs'); resolve(data); diff --git a/app/db/ip.js b/app/db/ip.js index 9b68166..db60fe2 100644 --- a/app/db/ip.js +++ b/app/db/ip.js @@ -2,6 +2,18 @@ const config = require('config'); const promise = require('bluebird'); const logger = require('../logger'); const db = require('./connection'); +const moment = require('moment'); + +const date2string_def = { + 'month': { + 'full': 'YYYY-MM-01 00:00:00', + 'header': 'YYYY-MM' + }, + 'day': { + 'full': 'YYYY-MM-DD 00:00:00', + 'header': 'YYYY-MM-DD' + } +}; exports.get = () => { logger.trace(); @@ -45,6 +57,143 @@ exports.get = () => { }); }; + +function date2string(date=new Date(), level='month', type='full'){ + return date.format(date2string_def[level][type]) +} + + +function createFilter(filter){ + + logger.warn('IP filter is not implemented'); + var query={ + 'pivot_ip': { + 'filter_ip' : '', + 'filter_period' : '', + 'filter_service' : ' AND lg.service_id IS NULL ' + }, + 'pivot_val': { // change to row !!! + 'filter_service' : ' AND data.service_id IS NULL ' + }, + 'pivot_col': { + 'names': ' ip INET ', + 'values': '' + } + }; + var values={}; + var header=['ip']; + + values.level = filter.period.level; + values.period_interval = '1 ' + values.level; + query.pivot_ip.filter_period = ' AND lg.period_start_date >= \'$\' AND lg.period_end_date < \'$\' '; + values.period_start = date2string(filter.period.start,filter.period.level,'full'); + values.period_end = date2string(filter.period.end,filter.period.level,'full'); + query.pivot_col.values = `generate_series( + date_trunc(\'$\', timestamp \'$\'), + date_trunc(\'$\', timestamp \'$\' - interval \'$\'), + \'$\'::interval + )`; + + var date_i = filter.period.start; + while(date_i < filter.period.end){ + var date_str = date2string(date_i,filter.period.level,'header'); + query.pivot_col.names += ', "'+date_str+'" BIGINT '; + header.push(date_str); + date_i = date_i.add(1,filter.period.level+'s'); + } + if(typeof filter.service_id !== 'undefined'){ + logger.warn('result contains only single/all services'); + query.pivot_ip.filter_service = ' AND lg.service_id = $ '; + query.pivot_val.filter_service = ' AND data.service_id = $ '; + values.service_id = filter.service_id; + } + + return { + query: query, + values: values, + header: header + } +} + + + +exports.getTop = (filter={}, + period_start = (new Date().getFullYear())+'-01-01 00:00:00', + period_end = (new Date().getFullYear() + 1)+'-01-01 00:00:00', + measure='units', + level='month', + min_exist = 0 + ) => { + logger.trace(); + logger.warn('db.ip.getTop() uses only default settings !!!'); + const {query, values, header} = createFilter({ + 'period': { + 'start': moment(period_start), + 'end': moment(period_end), + 'level': level + }, + 'value':{ + 'measure': measure, + 'min_exist': min_exist + }, + ...filter + }); + return new promise((resolve, reject) => { + db.any(` +SELECT * +FROM crosstab( + 'SELECT top.ip,to_char(data.period_start_date,''YYYY-MM-DD 00:00:00''),COALESCE(data.cnt_${measure},0) + FROM + ( SELECT DISTINCT ip + FROM log_ip_aggr lg + WHERE + lg.cnt_${measure} > $ + AND lg.period_level=\'$\'::period_levels + ${query.pivot_ip.filter_ip} + ${query.pivot_ip.filter_period} -- AND lg.period_start_date >= ''2021-01-01 00:00:00'' AND lg.period_end_date < ''2022-01-01 00:00:00'' + ${query.pivot_ip.filter_service} -- AND lg.service_id IS NULL + ) AS top + JOIN log_ip_aggr data + ON data.ip = top.ip + WHERE data.period_level=\'$\'::period_levels + ${query.pivot_val.filter_service} -- AND data.service_id IS NULL + ORDER BY 1,2', + 'SELECT s::timestamp + FROM + ${query.pivot_col.values} s' + ) as pivot ( + -- column names + ${query.pivot_col.names} + ); + `, { + 'min_exist': min_exist, + 'filter': values + }) + .then(data => { + logger.trace(); + if (data) { + resolve({data: data, header: header}); // data + } + else { + reject({ + state: 'failure', + reason: 'No IPs returned', + extra: null + }); + } + }) + .catch(error => { + logger.trace(); + logger.error(error); + reject({ + state: 'failure', + reason: 'Database error', + extra: error + }); + }); + }); +}; + exports.getRecent = () => { /* return only last three months*/ logger.trace(); return new promise((resolve, reject) => { diff --git a/app/routes/apiRoutes.js b/app/routes/apiRoutes.js index b84f730..0f51ddd 100644 --- a/app/routes/apiRoutes.js +++ b/app/routes/apiRoutes.js @@ -305,8 +305,8 @@ router.get('/api/ips', function (req, res, next) { error : 'Permission Denied.' }); } else { - dataController.getIPs(user.user_id).then(data => { - res.json({"data": data}); + dataController.getTopIPs(user.user_id).then(data => { + res.json(data); }).catch(); } }); diff --git a/app/views/ips.pug b/app/views/ips.pug index efb9652..e3f72d8 100644 --- a/app/views/ips.pug +++ b/app/views/ips.pug @@ -18,13 +18,12 @@ block content .row .col.s12 h6 Last three months IP list - table.striped(type='lb-datatable' data-type='json' data-field='data' data-url='api/ips' data-header='["IP", "Requests", "Units", "Requests", "Units", "Requests", "Units", "Requests", "Units"]' data-items='["ip", "m2_requests", "m2_units", "m1_requests", "m1_units", "m0_requests", "m0_units", "requests", "units"]' ) + table.striped(type='lb-datatable' data-type='json' data-field='data' data-url='api/ips' data-header-field='header' data-items-field='header' ) script. [ { "targets": 0, "data": null, "render": function ( data, type, row, meta ) {return 'settings_input_component '+data[0]+'';} } - ] diff --git a/scripts/database-patch02.sh b/scripts/database-patch02.sh index e3a43c2..494f67f 100755 --- a/scripts/database-patch02.sh +++ b/scripts/database-patch02.sh @@ -87,6 +87,7 @@ LANGUAGE plpgsql; CREATE TRIGGER user_endpoints_copy_ip_aggr AFTER INSERT ON user_endpoints FOR ROW EXECUTE PROCEDURE trigger_add_endpoint_copy_ip_aggr(); +CREATE EXTENSION IF NOT EXISTS tablefunc; INSERT INTO db_schema_version(version, script_path) VALUES(2, 'scripts/database-patch02.sh') ON CONFLICT DO NOTHING;