Skip to content

Commit

Permalink
[feature] Add a check which inspects device configuration status peri…
Browse files Browse the repository at this point in the history
…odically #54
  • Loading branch information
nepython committed Jun 12, 2020
1 parent 4b849dd commit 58539d6
Show file tree
Hide file tree
Showing 21 changed files with 455 additions and 89 deletions.
41 changes: 41 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,47 @@ in terms of disk space.

Whether ping checks are created automatically for devices.

``OPENWISP_MONITORING_AUTO_CONFIG_STATUS``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+--------------+-------------+
| **type**: | ``bool`` |
+--------------+-------------+
| **default**: | ``True`` |
+--------------+-------------+

Whether config_status checks are created automatically for devices.

``OPENWISP_MONITORING_CONFIG_MODIFIED_MAX_TIME``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+--------------+-----------+
| **type**: | ``float`` |
+--------------+-----------+
| **default**: | ``5`` |
+--------------+-----------+

After ``config`` is ``modified``, if the ``modified`` status does not change after a
fixed **duration** then ``device`` health status changes to ``problem``.
This **duration** can be set with the help of this setting. The input represents the duration in minutes.
Thus, by default the health status of the device changes to ``problem`` after 5 minutes of ``config_modified`` status.

**Note**: If the setting ``AUTO_CONFIG_STATUS`` is disabled then this setting need not be declared.

``OPENWISP_MONITORING_CONFIG_STATUS_RETENTION_POLICY``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+--------------+-------------+
| **type**: | ``time`` |
+--------------+-------------+
| **default**: | ``48h0m0s`` |
+--------------+-------------+

This setting allows to modify the duration for which the metric data generated
by ``modified_status`` check is to be retained.

**Note**: If the setting ``AUTO_CONFIG_STATUS`` is disabled then this setting need not be declared.

``OPENWISP_MONITORING_AUTO_GRAPHS``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Expand Down
5 changes: 5 additions & 0 deletions openwisp_monitoring/check/apps.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _

from .utils import manage_config_status_retention_policy


class CheckConfig(AppConfig):
name = 'openwisp_monitoring.check'
label = 'check'
verbose_name = _('Network Monitoring Checks')

def ready(self):
manage_config_status_retention_policy()
63 changes: 41 additions & 22 deletions openwisp_monitoring/check/base/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db import models, transaction
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.functional import cached_property
from django.utils.module_loading import import_string
from django.utils.translation import ugettext_lazy as _
Expand Down Expand Up @@ -75,27 +76,45 @@ def perform_check(self, store=True):
return self.check_instance.check(store=True)


if app_settings.AUTO_PING:
from django.db import transaction
from django.dispatch import receiver
@receiver(post_save, sender=Device, dispatch_uid='auto_ping')
def auto_ping_receiver(sender, instance, created, **kwargs):
"""
Implements OPENWISP_MONITORING_AUTO_PING
The creation step is executed in the background
"""
# we need to skip this otherwise this task will be executed
# every time the configuration is requested via checksum
from openwisp_monitoring.check.tasks import auto_create_ping

@receiver(post_save, sender=Device, dispatch_uid='auto_ping')
def auto_ping_receiver(sender, instance, created, **kwargs):
"""
Implements OPENWISP_MONITORING_AUTO_PING
The creation step is executed in the backround
"""
# we need to skip this otherwise this task will be executed
# every time the configuration is requested via checksum
if not created:
return
with transaction.atomic():
transaction.on_commit(
lambda: auto_create_ping.delay(
model=sender.__name__.lower(),
app_label=sender._meta.app_label,
object_id=str(instance.pk),
created=created,
)
if not app_settings.AUTO_PING or not created:
return
with transaction.atomic():
transaction.on_commit(
lambda: auto_create_ping.delay(
model=sender.__name__.lower(),
app_label=sender._meta.app_label,
object_id=str(instance.pk),
)
)


@receiver(post_save, sender=Device, dispatch_uid='auto_config_status')
def auto_config_status_receiver(sender, instance, created, **kwargs):
"""
Implements OPENWISP_MONITORING_AUTO_CONFIG_STATUS
The creation step is executed in the background
"""
# we need to skip this otherwise this task will be executed
# every time the configuration is requested via checksum
from openwisp_monitoring.check.tasks import auto_create_config_status

if not app_settings.AUTO_CONFIG_STATUS or not created:
return
with transaction.atomic():
transaction.on_commit(
lambda: auto_create_config_status.delay(
model=sender.__name__.lower(),
app_label=sender._meta.app_label,
object_id=str(instance.pk),
)
)
1 change: 1 addition & 0 deletions openwisp_monitoring/check/classes/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
from .config_status import ConfigStatus # noqa
from .ping import Ping # noqa
39 changes: 39 additions & 0 deletions openwisp_monitoring/check/classes/base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from swapper import load_model

from openwisp_controller.config.models import Device

Metric = load_model('monitoring', 'Metric')


class BaseCheck(object):
def validate_instance(self):
# check instance is of type device
obj = self.related_object
if not obj or not isinstance(obj, Device):
message = 'A related device is required to perform this operation'
raise ValidationError({'content_type': message, 'object_id': message})

def _get_or_create_metric(self, field_name):
"""
Gets or creates metric
"""
check = self.check_instance
if check.object_id and check.content_type:
obj_id = check.object_id
ct = check.content_type
else:
obj_id = str(check.id)
ct = ContentType.objects.get(
app_label=check._meta.app_label, model=check.__class__.__name__.lower()
)
options = dict(
name=check.name,
object_id=obj_id,
content_type=ct,
field_name=field_name,
key=self.__class__.__name__.lower(),
)
metric, created = Metric.objects.get_or_create(**options)
return metric, created
69 changes: 69 additions & 0 deletions openwisp_monitoring/check/classes/config_status.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
from time import time

from swapper import load_model

from ...monitoring.utils import write
from ..settings import CONFIG_MODIFIED_MAX_TIME
from ..utils import CONFIG_STATUS_RP
from .base import BaseCheck

Graph = load_model('monitoring', 'Graph')
Threshold = load_model('monitoring', 'Threshold')


class ConfigStatus(BaseCheck):
def __init__(self, check, params):
self.check_instance = check
self.related_object = check.content_object
self.params = params

def validate(self):
self.validate_instance()

def check(self, store=True):
if not hasattr(self.related_object, 'config'):
return
result = 0 if self.related_object.config.status == 'applied' else 1
if result and self._check_modified_status_crossed_max_time():
# TODO: When is the status supposed to be made critical?
dm = self.related_object.monitoring
if dm.status in ['ok', 'unknown']:
dm.update_status('problem')
if store:
metric = self.get_metric()
# TODO: Find why doing the same by adding rp as an arg
# in metric write changes status to problem
write(
name=metric.key,
values={metric.field_name: result},
tags=metric.tags,
retention_policy=CONFIG_STATUS_RP,
)
return result

# TODO: Check if this can be done using threshold
def _check_modified_status_crossed_max_time(self):
# A small margin is kept to take care of border cases
# TODO: Find why `m` --> minute is not working correctly!
since = f'now() - {int(CONFIG_MODIFIED_MAX_TIME*63)}s'
measurement_list = self.get_metric().read(
since=since, limit=None, order='time DESC'
)
for measurement in measurement_list:
if not measurement['config_status']:
break
minutes_difference = (time() - measurement['time']) / 60
if minutes_difference >= CONFIG_MODIFIED_MAX_TIME:
return True
return False

def get_metric(self):
metric, created = self._get_or_create_metric(field_name='config_status')
if created:
self._create_threshold(metric)
return metric

def _create_threshold(self, metric):
t = Threshold(metric=metric, operator='>', value=0, seconds=0)
t.full_clean()
t.save()
34 changes: 4 additions & 30 deletions openwisp_monitoring/check/classes/ping.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,22 @@
import subprocess

from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from jsonschema import draft7_format_checker, validate
from jsonschema.exceptions import ValidationError as SchemaError
from swapper import load_model

from openwisp_controller.config.models import Device

from ... import settings as monitoring_settings
from .. import settings as app_settings
from ..exceptions import OperationalError
from .base import BaseCheck

Graph = load_model('monitoring', 'Graph')
Metric = load_model('monitoring', 'Metric')
Threshold = load_model('monitoring', 'Threshold')


class Ping(object):
class Ping(BaseCheck):
schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'$schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
'additionalProperties': False,
'properties': {
Expand Down Expand Up @@ -57,13 +54,6 @@ def validate(self):
self.validate_instance()
self.validate_params()

def validate_instance(self):
# check instance is of type device
obj = self.related_object
if not obj or not isinstance(obj, Device):
message = 'A related device is required ' 'to perform this operation'
raise ValidationError({'content_type': message, 'object_id': message})

def validate_params(self):
try:
validate(self.params, self.schema, format_checker=draft7_format_checker)
Expand Down Expand Up @@ -165,23 +155,7 @@ def _get_metric(self):
"""
Gets or creates metric
"""
check = self.check_instance
if check.object_id and check.content_type:
obj_id = check.object_id
ct = check.content_type
else:
obj_id = str(check.id)
ct = ContentType.objects.get(
app_label=check._meta.app_label, model=check.__class__.__name__.lower()
)
options = dict(
name=check.name,
object_id=obj_id,
content_type=ct,
field_name='reachable',
key=self.__class__.__name__.lower(),
)
metric, created = Metric.objects.get_or_create(**options)
metric, created = self._get_or_create_metric(field_name='reachable')
if created:
self._create_threshold(metric)
self._create_graphs(metric)
Expand Down
4 changes: 3 additions & 1 deletion openwisp_monitoring/check/migrations/0001_initial.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import collections
import swapper

from ..settings import CHECK_CLASSES


class Migration(migrations.Migration):

Expand Down Expand Up @@ -58,7 +60,7 @@ class Migration(migrations.Migration):
(
'check',
models.CharField(
choices=[('openwisp_monitoring.check.classes.Ping', 'Ping')],
choices=CHECK_CLASSES,
db_index=True,
help_text='Select check type',
max_length=128,
Expand Down
1 change: 0 additions & 1 deletion openwisp_monitoring/check/migrations/0003_create_ping.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ def create_device_ping(apps, schema_editor):
model=Device.__name__.lower(),
app_label=Device._meta.app_label,
object_id=str(device.pk),
created=True,
)


Expand Down
33 changes: 33 additions & 0 deletions openwisp_monitoring/check/migrations/0004_create_config_status.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
from django.db import migrations
from openwisp_monitoring.check.settings import AUTO_CONFIG_STATUS
from openwisp_monitoring.check.tasks import auto_create_config_status


def add_config_status_checks(apps, schema_editor):
if not AUTO_CONFIG_STATUS:
return
Device = apps.get_model('config', 'Device')
for device in Device.objects.all():
auto_create_config_status.delay(
model=Device.__name__.lower(),
app_label=Device._meta.app_label,
object_id=str(device.pk),
)


def remove_config_status_checks(apps, schema_editor):
Check = apps.get_model('config', 'Device')
Check.objects.filter(name='Config Status').delete()


class Migration(migrations.Migration):

dependencies = [
('check', '0003_create_ping'),
]

operations = [
migrations.RunPython(
add_config_status_checks, reverse_code=remove_config_status_checks
),
]
Loading

0 comments on commit 58539d6

Please sign in to comment.