diff --git a/openwisp_monitoring/check/apps.py b/openwisp_monitoring/check/apps.py index 15c9832f0..1e5338e44 100644 --- a/openwisp_monitoring/check/apps.py +++ b/openwisp_monitoring/check/apps.py @@ -32,3 +32,12 @@ def _connect_signals(self): sender=load_model('config', 'Device'), dispatch_uid='auto_config_check', ) + + if app_settings.AUTO_SNMP_DEVICEMONITORING: + from .base.models import auto_snmp_devicemonitoring_receiver + + post_save.connect( + auto_snmp_devicemonitoring_receiver, + sender=load_model('config', 'Device'), + dispatch_uid='auto_snmp_devicemonitoring', + ) diff --git a/openwisp_monitoring/check/base/models.py b/openwisp_monitoring/check/base/models.py index a0bab9e66..4715570c4 100644 --- a/openwisp_monitoring/check/base/models.py +++ b/openwisp_monitoring/check/base/models.py @@ -9,7 +9,11 @@ from jsonfield import JSONField from openwisp_monitoring.check import settings as app_settings -from openwisp_monitoring.check.tasks import auto_create_config_check, auto_create_ping +from openwisp_monitoring.check.tasks import ( + auto_create_config_check, + auto_create_ping, + auto_create_snmp_devicemonitoring, +) from openwisp_utils.base import TimeStampedEditableModel from ...utils import transaction_on_commit @@ -116,3 +120,21 @@ def auto_config_check_receiver(sender, instance, created, **kwargs): object_id=str(instance.pk), ) ) + + +def auto_snmp_devicemonitoring_receiver(sender, instance, created, **kwargs): + """ + Implements OPENWISP_MONITORING_AUTO_DEVICE_SNMP_DEVICEMONITORING + The creation step is executed in the background + """ + # we need to skip this otherwise this task will be executed + # every time the configuration is requested via checksum + if not created: + return + transaction_on_commit( + lambda: auto_create_snmp_devicemonitoring.delay( + model=sender.__name__.lower(), + app_label=sender._meta.app_label, + object_id=str(instance.pk), + ) + ) diff --git a/openwisp_monitoring/check/classes/__init__.py b/openwisp_monitoring/check/classes/__init__.py index 33bf8293c..ff9cb43ac 100644 --- a/openwisp_monitoring/check/classes/__init__.py +++ b/openwisp_monitoring/check/classes/__init__.py @@ -1,2 +1,3 @@ from .config_applied import ConfigApplied # noqa from .ping import Ping # noqa +from .snmp_devicemonitoring import SnmpDeviceMonitoring # noqa diff --git a/openwisp_monitoring/check/classes/base.py b/openwisp_monitoring/check/classes/base.py index 71ef7582d..c24b4c7ce 100644 --- a/openwisp_monitoring/check/classes/base.py +++ b/openwisp_monitoring/check/classes/base.py @@ -5,6 +5,7 @@ Check = load_model('check', 'Check') Metric = load_model('monitoring', 'Metric') Device = load_model('config', 'Device') +DeviceData = load_model('device_monitoring', 'DeviceData') class BaseCheck(object): diff --git a/openwisp_monitoring/check/classes/snmp_devicemonitoring.py b/openwisp_monitoring/check/classes/snmp_devicemonitoring.py new file mode 100644 index 000000000..ec49cb122 --- /dev/null +++ b/openwisp_monitoring/check/classes/snmp_devicemonitoring.py @@ -0,0 +1,227 @@ +from copy import deepcopy + +from django.contrib.contenttypes.models import ContentType +from django.utils.functional import cached_property +from netengine.backends.snmp.openwrt import OpenWRT +from swapper import load_model + +from ... import settings as monitoring_settings +from .. import settings as app_settings +from .base import BaseCheck + +Chart = load_model('monitoring', 'Chart') +Metric = load_model('monitoring', 'Metric') +Device = load_model('config', 'Device') +AlertSettings = load_model('monitoring', 'AlertSettings') + + +class SnmpDeviceMonitoring(BaseCheck): + def check(self, store=True): + result = self.netengine_instance.to_dict() + self._init_previous_data() + self.related_object.data = result + if store: + self.store_result(result) + return result + + def store_result(self, data): + """ + store result in the DB + """ + ct = ContentType.objects.get_for_model(Device) + pk = self.related_object.pk + for interface in data.get('interfaces', []): + ifname = interface['name'] + ifstats = interface.get('statistics', {}) + # Explicitly stated None to avoid skipping in case the stats are zero + if ( + ifstats.get('rx_bytes') is not None + and ifstats.get('rx_bytes') is not None + ): + field_value = self._calculate_increment( + ifname, 'rx_bytes', ifstats['rx_bytes'] + ) + extra_values = { + 'tx_bytes': self._calculate_increment( + ifname, 'tx_bytes', ifstats['tx_bytes'] + ) + } + name = f'{ifname} traffic' + metric, created = Metric._get_or_create( + object_id=pk, + content_type=ct, + configuration='traffic', + name=name, + key=ifname, + ) + metric.write(field_value, extra_values=extra_values) + if created: + self._create_traffic_chart(metric) + try: + clients = interface['wireless']['clients'] + except KeyError: + continue + if not isinstance(clients, list): + continue + name = '{0} wifi clients'.format(ifname) + metric, created = Metric._get_or_create( + object_id=pk, + content_type=ct, + configuration='clients', + name=name, + key=ifname, + ) + for client in clients: + if 'mac' not in client: + continue + metric.write(client['mac']) + if created: + self._create_clients_chart(metric) + if 'resources' not in data: + return + if 'load' in data['resources'] and 'cpus' in data['resources']: + self._write_cpu( + data['resources']['load'], data['resources']['cpus'], pk, ct + ) + if 'disk' in data['resources']: + self._write_disk(data['resources']['disk'], pk, ct) + if 'memory' in data['resources']: + self._write_memory(data['resources']['memory'], pk, ct) + + @cached_property + def netengine_instance(self): + params = self.params['credential_params'] + ip = self._get_ip() + return OpenWRT(host=ip, **params) + + def _get_ip(self): + """ + Figures out ip to use or fails raising OperationalError + """ + device = self.related_object + ip = device.management_ip + if not ip and not app_settings.MANAGEMENT_IP_ONLY: + ip = device.last_ip + return ip + + def _write_cpu(self, load, cpus, primary_key, content_type): + extra_values = { + 'load_1': float(load[0]), + 'load_5': float(load[1]), + 'load_15': float(load[2]), + } + metric, created = Metric._get_or_create( + object_id=primary_key, content_type=content_type, configuration='cpu' + ) + if created: + self._create_resources_chart(metric, resource='cpu') + self._create_resources_alert_settings(metric, resource='cpu') + metric.write(100 * float(load[0] / cpus), extra_values=extra_values) + + def _write_disk(self, disk_list, primary_key, content_type): + used_bytes, size_bytes, available_bytes = 0, 0, 0 + for disk in disk_list: + used_bytes += disk['used_bytes'] + size_bytes += disk['size_bytes'] + available_bytes += disk['available_bytes'] + metric, created = Metric._get_or_create( + object_id=primary_key, content_type=content_type, configuration='disk' + ) + if created: + self._create_resources_chart(metric, resource='disk') + self._create_resources_alert_settings(metric, resource='disk') + metric.write(100 * used_bytes / size_bytes) + + def _write_memory(self, memory, primary_key, content_type): + extra_values = { + 'total_memory': memory['total'], + 'free_memory': memory['free'], + 'buffered_memory': memory['shared'], + 'shared_memory': memory['shared'], + } + if 'cached' in memory: + extra_values['cached_memory'] = memory.get('cached') + percent_used = 100 * (1 - (memory['free'] + memory['shared']) / memory['total']) + # Available Memory is not shown in some systems (older openwrt versions) + if 'available' in memory: + extra_values.update({'available_memory': memory['available']}) + if memory['available'] > memory['free']: + percent_used = 100 * ( + 1 - (memory['available'] + memory['shared']) / memory['total'] + ) + metric, created = Metric._get_or_create( + object_id=primary_key, content_type=content_type, configuration='memory' + ) + if created: + self._create_resources_chart(metric, resource='memory') + self._create_resources_alert_settings(metric, resource='memory') + metric.write(percent_used, extra_values=extra_values) + + def _calculate_increment(self, ifname, stat, value): + """ + compares value with previously stored counter and + calculates the increment of the value (which is returned) + """ + # get previous counters + data = self._previous_data + try: + previous_counter = data['interfaces_dict'][ifname]['statistics'][stat] + except KeyError: + # if no previous measurements present, counter will start from zero + previous_counter = 0 + # if current value is higher than previous value, + # it means the interface traffic counter is increasing + # and to calculate the traffic performed since the last + # measurement we have to calculate the difference + if value >= previous_counter: + return value - previous_counter + # on the other side, if the current value is less than + # the previous value, it means that the counter was restarted + # (eg: reboot, configuration reload), so we keep the whole amount + else: + return value + + def _create_traffic_chart(self, metric): + """ + create "traffic (GB)" chart + """ + if 'traffic' not in monitoring_settings.AUTO_CHARTS: + return + chart = Chart(metric=metric, configuration='traffic') + chart.full_clean() + chart.save() + + def _create_clients_chart(self, metric): + """ + creates "WiFi associations" chart + """ + if 'wifi_clients' not in monitoring_settings.AUTO_CHARTS: + return + chart = Chart(metric=metric, configuration='wifi_clients') + chart.full_clean() + chart.save() + + def _create_resources_chart(self, metric, resource): + if resource not in monitoring_settings.AUTO_CHARTS: + return + chart = Chart(metric=metric, configuration=resource) + chart.full_clean() + chart.save() + + def _create_resources_alert_settings(self, metric, resource): + alert_settings = AlertSettings(metric=metric) + alert_settings.full_clean() + alert_settings.save() + + def _init_previous_data(self): + """ + makes NetJSON interfaces of previous + snapshots more easy to access + """ + data = getattr(self.related_object, 'data', {}) + if data: + data = deepcopy(data) + data['interfaces_dict'] = {} + for interface in data.get('interfaces', []): + data['interfaces_dict'][interface['name']] = interface + self._previous_data = data diff --git a/openwisp_monitoring/check/settings.py b/openwisp_monitoring/check/settings.py index 4575c8eca..7a8389a68 100644 --- a/openwisp_monitoring/check/settings.py +++ b/openwisp_monitoring/check/settings.py @@ -5,9 +5,17 @@ ( ('openwisp_monitoring.check.classes.Ping', 'Ping'), ('openwisp_monitoring.check.classes.ConfigApplied', 'Configuration Applied'), + ( + 'openwisp_monitoring.check.classes.SnmpDeviceMonitoring', + 'SNMP Device Monitoring', + ), ), ) -AUTO_PING = get_settings_value('AUTO_PING', True) -AUTO_CONFIG_CHECK = get_settings_value('AUTO_DEVICE_CONFIG_CHECK', True) -MANAGEMENT_IP_ONLY = get_settings_value('MANAGEMENT_IP_ONLY', True) -PING_CHECK_CONFIG = get_settings_value('PING_CHECK_CONFIG', {}) +AUTO_PING = getattr(settings, 'OPENWISP_MONITORING_AUTO_PING', True) +AUTO_SNMP_DEVICEMONITORING = getattr( + settings, 'OPENWISP_MONITORING_AUTO_SNMP_DEVICEMONITORING', True +) +AUTO_CONFIG_CHECK = getattr( + settings, 'OPENWISP_MONITORING_AUTO_DEVICE_CONFIG_CHECK', True +) +MANAGEMENT_IP_ONLY = getattr(settings, 'OPENWISP_MONITORING_MANAGEMENT_IP_ONLY', True) diff --git a/openwisp_monitoring/check/tasks.py b/openwisp_monitoring/check/tasks.py index 2ae62bc0d..530f54e6d 100644 --- a/openwisp_monitoring/check/tasks.py +++ b/openwisp_monitoring/check/tasks.py @@ -7,6 +7,8 @@ from django.core.exceptions import ObjectDoesNotExist from swapper import load_model +from openwisp_controller.connection import settings as app_settings + logger = logging.getLogger(__name__) @@ -14,6 +16,31 @@ def get_check_model(): return load_model('check', 'Check') +def _get_or_create_credentials(device_id, **kwargs): + Credentials = load_model('connection', 'Credentials') + cred = Credentials.objects.filter( + deviceconnection__device_id=device_id, + connector='openwisp_controller.connection.connectors.snmp.Snmp', + ).last() + if cred is not None: + return cred + + # if credentials don't exist, create new SNMP credentials + Device = load_model('config', 'Device') + opts = dict( + name='Default SNMP Credentials', + connector=app_settings.DEFAULT_CONNECTORS[1][0], + params={'community': 'public', 'agent': 'default', 'port': 161}, + ) + opts.update(kwargs) + if 'organization' not in opts: + opts['organization'] = Device.objects.get(id=device_id).organization + c = Credentials(**opts) + c.full_clean() + c.save() + return c + + @shared_task def run_checks(): """ @@ -100,3 +127,32 @@ def auto_create_config_check( ) check.full_clean() check.save() + + +@shared_task +def auto_create_snmp_devicemonitoring( + model, app_label, object_id, check_model=None, content_type_model=None +): + """ + Called by openwisp_monitoring.check.models.auto_snmp_devicemonitoring_receiver + """ + Check = check_model or get_check_model() + devicemonitoring_path = 'openwisp_monitoring.check.classes.SnmpDeviceMonitoring' + has_check = Check.objects.filter( + object_id=object_id, content_type__model='device', check=devicemonitoring_path + ).exists() + # create new check only if necessary + if has_check: + return + content_type_model = content_type_model or ContentType + ct = content_type_model.objects.get(app_label=app_label, model=model) + cred = _get_or_create_credentials(object_id) + check = Check( + name='SNMP Device Monitoring', + check=devicemonitoring_path, + content_type=ct, + object_id=object_id, + params={'credential_params': cred.get_params()}, + ) + check.full_clean() + check.save() diff --git a/openwisp_monitoring/check/tests/test_models.py b/openwisp_monitoring/check/tests/test_models.py index abbf8ed13..3f645c2fc 100644 --- a/openwisp_monitoring/check/tests/test_models.py +++ b/openwisp_monitoring/check/tests/test_models.py @@ -9,8 +9,12 @@ from ...device.tests import TestDeviceMonitoringMixin from .. import settings as app_settings -from ..classes import ConfigApplied, Ping -from ..tasks import auto_create_config_check, auto_create_ping +from ..classes import ConfigApplied, Ping, SnmpDeviceMonitoring +from ..tasks import ( + auto_create_config_check, + auto_create_ping, + auto_create_snmp_devicemonitoring, +) Check = load_model('check', 'Check') Metric = load_model('monitoring', 'Metric') @@ -22,6 +26,7 @@ class TestModels(TestDeviceMonitoringMixin, TransactionTestCase): _PING = app_settings.CHECK_CLASSES[0][0] _CONFIG_APPLIED = app_settings.CHECK_CLASSES[1][0] + _SNMP_DEVICEMONITORING = app_settings.CHECK_CLASSES[2][0] def test_check_str(self): c = Check(name='Test check') @@ -48,6 +53,12 @@ def test_check_class(self): check_type=self._CONFIG_APPLIED, ) self.assertEqual(c.check_class, ConfigApplied) + with self.subTest('Test DeviceMonitoring check Class'): + c = Check( + name='SnmpDeviceMonitoring class check', + check=self._SNMP_DEVICEMONITORING, + ) + self.assertEqual(c.check_class, SnmpDeviceMonitoring) def test_base_check_class(self): path = 'openwisp_monitoring.check.classes.base.BaseCheck' @@ -81,6 +92,17 @@ def test_check_instance(self): self.assertIsInstance(i, ConfigApplied) self.assertEqual(i.related_object, obj) self.assertEqual(i.params, c.params) + with self.subTest('Test DeviceMonitoring check instance'): + c = Check( + name='DeviceMonitoring class check', + check=self._SNMP_DEVICEMONITORING, + content_object=obj, + params={}, + ) + i = c.check_instance + self.assertIsInstance(i, SnmpDeviceMonitoring) + self.assertEqual(i.related_object, obj) + self.assertEqual(i.params, c.params) def test_validation(self): with self.subTest('Test Ping check validation'): @@ -105,7 +127,7 @@ def test_validation(self): def test_auto_check_creation(self): self.assertEqual(Check.objects.count(), 0) d = self._create_device(organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) with self.subTest('Test AUTO_PING'): c1 = Check.objects.filter(check_type=self._PING).first() self.assertEqual(c1.content_object, d) @@ -118,7 +140,7 @@ def test_auto_check_creation(self): def test_device_deleted(self): self.assertEqual(Check.objects.count(), 0) d = self._create_device(organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) d.delete() self.assertEqual(Check.objects.count(), 0) @@ -129,7 +151,7 @@ def test_config_modified_device_problem(self): self._create_config(status='modified', organization=self._create_org()) d = Device.objects.first() d.monitoring.update_status('ok') - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) self.assertEqual(Metric.objects.count(), 0) self.assertEqual(AlertSettings.objects.count(), 0) check = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() @@ -159,7 +181,7 @@ def test_config_error(self): self._create_config(status='error', organization=self._create_org()) dm = Device.objects.first().monitoring dm.update_status('ok') - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) self.assertEqual(Metric.objects.count(), 0) self.assertEqual(AlertSettings.objects.count(), 0) check = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() @@ -192,7 +214,7 @@ def test_config_error(self): @patch('openwisp_monitoring.check.settings.AUTO_PING', False) def test_config_check_critical_metric(self): self._create_config(status='modified', organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) d = Device.objects.first() dm = d.monitoring dm.update_status('ok') @@ -211,7 +233,7 @@ def test_config_check_critical_metric(self): def test_no_duplicate_check_created(self): self._create_config(organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) d = Device.objects.first() auto_create_config_check.delay( model=Device.__name__.lower(), @@ -223,14 +245,19 @@ def test_no_duplicate_check_created(self): app_label=Device._meta.app_label, object_id=str(d.pk), ) - self.assertEqual(Check.objects.count(), 2) + auto_create_snmp_devicemonitoring.delay( + model=Device.__name__.lower(), + app_label=Device._meta.app_label, + object_id=str(d.pk), + ) + self.assertEqual(Check.objects.count(), 3) def test_device_unreachable_no_config_check(self): self._create_config(status='modified', organization=self._create_org()) d = self.device_model.objects.first() d.monitoring.update_status('critical') - self.assertEqual(Check.objects.count(), 2) - c2 = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() + self.assertEqual(Check.objects.count(), 3) + c2 = Check.objects.filter(check=self._CONFIG_APPLIED).first() c2.perform_check() self.assertEqual(Metric.objects.count(), 0) self.assertIsNone(c2.perform_check()) @@ -240,8 +267,8 @@ def test_device_unknown_no_config_check(self): self._create_config(status='modified', organization=self._create_org()) d = self.device_model.objects.first() d.monitoring.update_status('unknown') - self.assertEqual(Check.objects.count(), 2) - c2 = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() + self.assertEqual(Check.objects.count(), 3) + c2 = Check.objects.filter(check=self._CONFIG_APPLIED).first() c2.perform_check() self.assertEqual(Metric.objects.count(), 0) self.assertEqual(Notification.objects.count(), 0) diff --git a/openwisp_monitoring/check/tests/test_ping.py b/openwisp_monitoring/check/tests/test_ping.py index 11b9ee47b..23b94652d 100644 --- a/openwisp_monitoring/check/tests/test_ping.py +++ b/openwisp_monitoring/check/tests/test_ping.py @@ -239,7 +239,7 @@ def test_store_result(self, mocked_method): device.management_ip = '10.40.0.1' device.save() # check created automatically by autoping - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) self.assertEqual(Metric.objects.count(), 0) self.assertEqual(Chart.objects.count(), 0) self.assertEqual(AlertSettings.objects.count(), 0) diff --git a/openwisp_monitoring/check/tests/test_utils.py b/openwisp_monitoring/check/tests/test_utils.py index c83e0993a..34c876c65 100644 --- a/openwisp_monitoring/check/tests/test_utils.py +++ b/openwisp_monitoring/check/tests/test_utils.py @@ -5,7 +5,7 @@ from swapper import load_model from ...device.tests import TestDeviceMonitoringMixin -from ..classes import Ping +from ..classes import Ping, SnmpDeviceMonitoring from ..settings import CHECK_CLASSES from ..tasks import perform_check from ..utils import run_checks_async @@ -24,12 +24,14 @@ def _create_check(self): # check is automatically created via django signal @patch.object(Ping, '_command', return_value=_FPING_REACHABLE) - def test_run_checks_async_success(self, mocked_method): + @patch.object(SnmpDeviceMonitoring, 'netengine_instance') + def test_run_checks_async_success(self, mocked_netengine, mocked_method): self._create_check() run_checks_async() + @patch.object(SnmpDeviceMonitoring, 'netengine_instance') @patch.object(Ping, '_command', return_value=_FPING_REACHABLE) - def test_management_command(self, mocked_method): + def test_management_command(self, mocked_method, mocked_netengine): self._create_check() management.call_command('run_checks') diff --git a/openwisp_monitoring/device/tests/test_transactions.py b/openwisp_monitoring/device/tests/test_transactions.py index 5fd3d9698..6b8f7c3b3 100644 --- a/openwisp_monitoring/device/tests/test_transactions.py +++ b/openwisp_monitoring/device/tests/test_transactions.py @@ -8,7 +8,7 @@ from openwisp_controller.connection.tests.base import CreateConnectionsMixin from openwisp_utils.tests import catch_signal -from ...check.classes import Ping +from ...check.classes import Ping, SnmpDeviceMonitoring from ...check.tests import _FPING_REACHABLE, _FPING_UNREACHABLE from ..tasks import trigger_device_checks from . import DeviceMonitoringTransactionTestcase @@ -54,10 +54,11 @@ def test_trigger_device_recovery_task(self, mocked_method): self._post_data(d.id, d.key, data) mock.assert_called_once() + @patch.object(SnmpDeviceMonitoring, 'netengine_instance') @patch.object(Ping, '_command', return_value=_FPING_UNREACHABLE) @patch.object(DeviceMonitoring, 'update_status') def test_trigger_device_recovery_task_regression( - self, mocked_update_status, mocked_ping + self, mocked_update_status, mocked_ping, mocked_snmp ): dm = self._create_device_monitoring() dm.device.management_ip = None diff --git a/requirements.txt b/requirements.txt index b6bd3450e..f1fe3da13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,12 @@ -openwisp-controller~=1.0.0 -influxdb~=5.3.1 +# TODO: put back the standard pypi release when +# next point version of openwisp-controller is released +openwisp-controller @ https://github.com/openwisp/openwisp-controller/tarball/issues/471-snmp-connections +# openwisp-utils[rest] is left unpinned so the version used +# is the one dictated by openwisp-controller +influxdb~=5.3 +django-celery-email~=3.0.0 django-cache-memoize~=0.1.0 django-nested-admin~=3.4.0 netaddr~=0.8 python-dateutil>=2.7,<3.0 -openwisp-utils[rest] @ https://github.com/openwisp/openwisp-utils/tarball/master +netengine @ https://github.com/openwisp/netengine/tarball/netjson-compliance