Merge pull request #80 from DanSheps/develop-3.6

3.6 support

netbox_config_backup/__init__.py

@@ -12,8 +12,8 @@ class NetboxConfigBackup(PluginConfig):
     author = metadata.get('Author')
     author_email = metadata.get('Author-email')
     base_url = 'configbackup'
-    min_version = '3.5.0'
-    max_version = '3.5.99'
+    min_version = '3.5.8'
+    max_version = '3.6.99'
     required_settings = [
         'repository',
         'committer',

netbox_config_backup/git.py

@@ -3,6 +3,7 @@
 from datetime import datetime
 from time import sleep
 
+from deepdiff import DeepDiff
 from dulwich import repo, porcelain, object_store
 from pydriller import Git
 

netbox_config_backup/management/commands/runbackup.py

@@ -1,17 +1,42 @@
-import datetime
+import uuid
+
 from django.core.management.base import BaseCommand
 from django.db import transaction
+from django.utils import timezone
 
+from netbox_config_backup.models import BackupJob
 from netbox_config_backup.tasks import backup_job
+from netbox_config_backup.utils import remove_queued
+from netbox_config_backup.utils.rq import can_backup
 
 
 class Command(BaseCommand):
     def add_arguments(self, parser):
-        parser.add_argument('--time', help="time")
-        parser.add_argument('device', help="Device Name")
+        parser.add_argument('--time', dest='time', help="time")
+        parser.add_argument('--device', dest='device', help="Device Name")
+
+    def run_backup(self, backup):
+        if can_backup(backup):
+            backupjob = backup.jobs.filter(backup__device=backup.device).last()
+            if backupjob is None:
+                backupjob = BackupJob.objects.create(
+                    backup=backup,
+                    scheduled=timezone.now(),
+                    uuid=uuid.uuid4()
+                )
+            backup_job(backupjob.pk)
+        remove_queued(backup)
 
     def handle(self, *args, **options):
-        from netbox_config_backup.models import Backup, BackupJob
+        from netbox_config_backup.models import Backup
+        if options['device']:
+            print(f'Running:{options.get("device")}| ')
+            backup = Backup.objects.filter(device__name=options['device']).first()
+            if backup:
+                self.run_backup(backup)
+            else:
+                raise Exception('Device not found')
+        else:
+            for backup in Backup.objects.all():
+                self.run_backup(backup)
 
-        backupjob = BackupJob.objects.filter(backup__device__name=options['device']).last()
-        backup_job(backupjob.pk)

netbox_config_backup/models/backups.py

@@ -3,6 +3,7 @@
 import uuid as uuid
 
 from django.db import models
+from django.db.models import Q
 from django.urls import reverse
 
 from django_rq import get_queue
@@ -89,7 +90,14 @@ def enqueue_if_needed(self):
         return enqueue_if_needed(self)
 
     def requeue(self):
-        self.jobs.all().delete()
+        self.jobs.filter(
+            ~Q(status=JobResultStatusChoices.STATUS_COMPLETED) &
+            ~Q(status=JobResultStatusChoices.STATUS_FAILED) &
+            ~Q(status=JobResultStatusChoices.STATUS_ERRORED)
+        ).update(
+            status=JobResultStatusChoices.STATUS_FAILED
+        )
+        remove_queued(self)
         self.enqueue_if_needed()
 
     def get_config(self, index='HEAD'):
@@ -226,13 +234,16 @@ class BackupCommitTreeChange(BigIDModel):
     type = models.CharField(max_length=10)
     old = models.ForeignKey(to=BackupObject, on_delete=models.PROTECT, related_name='previous', null=True)
     new = models.ForeignKey(to=BackupObject, on_delete=models.PROTECT, related_name='changes', null=True)
-
+    
     def __str__(self):
         return f'{self.commit.sha}-{self.type}'
 
     def filename(self):
         return f'{self.backup.uuid}.{self.type}'
 
+    def get_absolute_url(self):
+        return reverse('plugins:netbox_config_backup:backup_config', kwargs={'backup': self.backup.pk, 'current': self.pk})
+
     @property
     def previous(self):
         return self.backup.changes.filter(file__type=self.file.type, commit__time__lt=self.commit.time).last()

netbox_config_backup/tables.py

@@ -2,17 +2,17 @@
 from django_tables2.utils import Accessor
 
 from netbox_config_backup.models import Backup, BackupCommitTreeChange
-from netbox.tables import columns, BaseTable
+from netbox.tables import columns, BaseTable, NetBoxTable
 
 
 class ActionButtonsColumn(tables.TemplateColumn):
     attrs = {'td': {'class': 'text-end text-nowrap noprint min-width'}}
     template_code = """
-    <a href="{% url 'plugins:netbox_config_backup:backup_config' pk=record.backup.pk current=record.pk %}" class="btn btn-sm btn-outline-dark" title="View">
+    <a href="{% url 'plugins:netbox_config_backup:backup_config' backup=record.backup.pk current=record.pk %}" class="btn btn-sm btn-outline-dark" title="View">
         <i class="mdi mdi-cloud-download"></i>
     </a>
     {% if record.previous %}
-        <a href="{% url 'plugins:netbox_config_backup:backup_diff' pk=record.backup.pk current=record.pk previous=record.previous.pk %}" class="btn btn-outline-dark btn-sm" title="Diff">
+        <a href="{% url 'plugins:netbox_config_backup:backup_diff' backup=record.backup.pk current=record.pk previous=record.previous.pk %}" class="btn btn-outline-dark btn-sm" title="Diff">
             <i class="mdi mdi-file-compare"></i>
         </a>
     {% else %}
@@ -28,7 +28,9 @@ def header(self):
 
 
 class BackupTable(BaseTable):
-    pk = columns.ToggleColumn()
+    pk = columns.ToggleColumn(
+
+    )
     name = tables.Column(
         linkify=True,
         verbose_name='Backup Name'
@@ -56,7 +58,7 @@ class Meta(BaseTable.Meta):
         )
 
 
-class BackupsTable(BaseTable):
+class BackupsTable(NetBoxTable):
     date = tables.Column(
         accessor='commit__time'
     )
@@ -68,10 +70,10 @@ class BackupsTable(BaseTable):
     class Meta:
         model = BackupCommitTreeChange
         fields = (
-            'date', 'type', 'backup', 'commit', 'file', 'actions'
+            'pk', 'id', 'date', 'type', 'backup', 'commit', 'file', 'actions'
         )
         default_columns = (
-            'date', 'type', 'actions'
+            'pk', 'id', 'date', 'type', 'actions'
         )
         attrs = {
             'class': 'table table-hover object-list',

netbox_config_backup/tasks.py

@@ -12,6 +12,8 @@
 from netbox.api.exceptions import ServiceUnavailable
 from netbox.config import get_config
 from netbox_config_backup.models import Backup, BackupJob, BackupCommit
+from netbox_config_backup.utils.rq import can_backup
+
 
 def get_logger():
     # Setup logging to Stdout
@@ -26,13 +28,14 @@ def get_logger():
 
 
 def napalm_init(device, ip=None, extra_args={}):
-    config = get_config()
-    username = config.NAPALM_USERNAME
-    password = config.NAPALM_PASSWORD
-    timeout = config.NAPALM_TIMEOUT
-    optional_args = config.NAPALM_ARGS.copy()
-    if device and device.platform and device.platform.napalm_args is not None:
-        optional_args.update(device.platform.napalm_args)
+    from netbox import settings
+    username = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_USERNAME', None)
+    password = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_PASSWORD', None)
+    timeout = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_TIMEOUT', None)
+    optional_args = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_ARGS', []).copy()
+
+    if device and device.platform and device.platform.napalm.napalm_args is not None:
+        optional_args.update(device.platform.napalm.napalm_args)
     if extra_args != {}:
         optional_args.update(extra_args)
 
@@ -57,10 +60,10 @@ def napalm_init(device, ip=None, extra_args={}):
 
     # Validate the configured driver
     try:
-        driver = napalm.get_network_driver(device.platform.napalm_driver)
+        driver = napalm.get_network_driver(device.platform.napalm.napalm_driver)
     except ModuleImportError:
         raise ServiceUnavailable("NAPALM driver for platform {} not found: {}.".format(
-            device.platform, device.platform.napalm_driver
+            device.platform, device.platform.napalm.napalm_driver
         ))
 
     # Connect to the device
@@ -89,6 +92,9 @@ def backup_config(backup, pk=None):
         ip = backup.ip if backup.ip is not None else backup.device.primary_ip
     else:
         ip = None
+    if not can_backup(backup):
+        raise Exception(f'Cannot backup {backup}')
+
     if backup.device is not None and ip is not None:
         logger.info(f'{backup}: Backup started')
         #logger.debug(f'[{pk}] Connecting')
@@ -119,6 +125,10 @@ def backup_job(pk):
         logger.error(f'Cannot locate job (Id: {pk}) in DB')
         raise Exception(f'Cannot locate job (Id: {pk}) in DB')
     backup = job_result.backup
+
+    if not can_backup(backup):
+        logger.warning(f'Cannot backup due to additional factors')
+        return 1
     delay = timedelta(seconds=settings.PLUGINS_CONFIG.get('netbox_config_backup', {}).get('frequency'))
 
     job_result.started = timezone.now()
@@ -135,33 +145,30 @@ def backup_job(pk):
         # Enqueue next job if one doesn't exist
         try:
             #logger.debug(f'[{pk}] Starting Enqueue')
+            BackupJob.objects.filter(
+                backup=backup
+            ).exclude(
+                status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
+            ).update(status=JobResultStatusChoices.STATUS_FAILED)
             BackupJob.enqueue_if_needed(backup, delay=delay, job_id=job_result.job_id)
             #logger.debug(f'[{pk}] Finished Enqueue')
         except Exception as e:
             logger.error(f'Job Enqueue after completion failed for job: {backup}')
             logger.error(f'\tException: {e}')
     except netmiko.exceptions.ReadTimeout as e:
         BackupJob.enqueue_if_needed(backup, delay=delay, job_id=job_result.job_id)
-        logger.error(f'Netmiko read timeout on job: {backup}')
+        logger.warning(f'Netmiko read timeout on job: {backup}')
     except ServiceUnavailable as e:
         logger.info(f'Napalm service read failure on job: {backup}')
         BackupJob.enqueue_if_needed(backup, delay=delay, job_id=job_result.job_id)
     except Exception as e:
         logger.error(f'Exception at line 148 on job: {backup}')
-        logger.error(e.with_traceback())
-        job_result.set_status(JobResultStatusChoices.STATUS_FAILED)
+        logger.error(e)
+        job_result.set_status(JobResultStatusChoices.STATUS_ERRORED)
         BackupJob.enqueue_if_needed(backup, delay=delay, job_id=job_result.job_id)
 
     #logger.debug(f'[{pk}] Saving result')
     job_result.save()
 
-    # Clear queue of old jobs
-    BackupJob.objects.filter(
-        backup=backup,
-        status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
-    ).exclude(
-        pk=job_result.pk
-    ).delete()
-
 
 logger = get_logger()

netbox_config_backup/templates/netbox_config_backup/backups.html

@@ -1,4 +1,4 @@
-{% extends 'generic/object.html' %}
+{% extends 'generic/object_children.html' %}
 {% load helpers %}
 
 {% block subtitle %}
@@ -19,18 +19,14 @@
   </div>
 {% endblock %}
 
-{% block content %}
-  {% include 'inc/table_controls_htmx.html' with table_modal="BackupsTable_config" %}
+{% block bulk_edit_controls %}
+    {{ block.super }}
+    {% with diff_view=object|viewname:"diff" %}
 
-    <div class="card">
-      <div class="card-body" id="object_list">
-        {% include 'htmx/table.html' %}
-      </div>
-    </div>
-  </form>
-{% endblock %}
-
-{% block modals %}
-  {{ block.super }}
-  {% table_config_form table %}
-{% endblock modals %}
+            <button type="submit" name="_diff"
+                    formaction="{% url diff_view backup=object.pk %}?return_url={{ return_url }}"
+                    class="btn btn-warning btn-sm">
+                <i class="mdi mdi-file-compare" aria-hidden="true"></i> Diff Selected (Max 2)
+            </button>
+    {% endwith %}
+{% endblock bulk_edit_controls %}

netbox_config_backup/urls.py

@@ -9,9 +9,9 @@
     path('devices/<int:pk>/edit/', views.BackupEditView.as_view(), name='backup_edit'),
     path('devices/<int:pk>/delete/', views.BackupDeleteView.as_view(), name='backup_delete'),
     path('devices/<int:pk>/backups/', views.BackupBackupsView.as_view(), name='backup_backups'),
-    path('devices/<int:pk>/config/', views.ConfigView.as_view(), name='backup_config'),
-    path('devices/<int:pk>/config/<int:current>/', views.ConfigView.as_view(), name='backup_config'),
-    path('devices/<int:pk>/diff/', views.DiffView.as_view(), name='backup_diff'),
-    path('devices/<int:pk>/diff/<int:current>/', views.DiffView.as_view(), name='backup_diff'),
-    path('devices/<int:pk>/diff/<int:current>/<int:previous>/', views.DiffView.as_view(), name='backup_diff'),
+    path('devices/<int:backup>/config/', views.ConfigView.as_view(), name='backup_config'),
+    path('devices/<int:backup>/config/<int:current>/', views.ConfigView.as_view(), name='backup_config'),
+    path('devices/<int:backup>/diff/', views.DiffView.as_view(), name='backup_diff'),
+    path('devices/<int:backup>/diff/<int:current>/', views.DiffView.as_view(), name='backup_diff'),
+    path('devices/<int:backup>/diff/<int:current>/<int:previous>/', views.DiffView.as_view(), name='backup_diff'),
 ]

netbox_config_backup/utils/rq.py

@@ -13,6 +13,37 @@
 logger = logging.getLogger(f"netbox_config_backup")
 
 
+def can_backup(backup):
+    logger.debug(f'Checking backup status for {backup}')
+    if backup.device is None:
+        logger.info(f'No device for {backup}')
+        return False
+    elif backup.status == StatusChoices.STATUS_DISABLED:
+        logger.info(f'Backup disabled for {backup}')
+        return False
+    elif backup.device.status in [DeviceStatusChoices.STATUS_OFFLINE,
+                                DeviceStatusChoices.STATUS_FAILED,
+                                DeviceStatusChoices.STATUS_INVENTORY,
+                                DeviceStatusChoices.STATUS_PLANNED]:
+        logger.info(f'Backup disabled for {backup} due to device status ({backup.device.status})')
+        return False
+    elif (backup.ip is None and backup.device.primary_ip is None) or backup.device.platform is None or \
+            hasattr(backup.device.platform, 'napalm') is False or backup.device.platform.napalm is None or \
+            backup.device.platform.napalm.napalm_driver == '' or backup.device.platform.napalm.napalm_driver is None:
+        if backup.ip is None and backup.device.primary_ip is None:
+            logger.warning(f'Backup disabled for {backup} due to no primary IP ({backup.device.status})')
+        elif backup.device.platform is None:
+            logger.warning(f'Backup disabled for {backup} due to platform not set ({backup.device.status})')
+        elif hasattr(backup.device.platform, 'napalm') is False or backup.device.platform.napalm is None:
+            logger.warning(
+                f'Backup disabled for {backup} due to platform having no napalm config ({backup.device.status})'
+            )
+        elif backup.device.platform.napalm.napalm_driver == '' or backup.device.platform.napalm.napalm_driver is None:
+            logger.warning(f'Backup disabled for {backup} due to napalm driver not set ({backup.device.status})')
+        return False
+
+    return True
+
 def enqueue(backup, delay=None):
     from netbox_config_backup.models import BackupJob
 
@@ -64,24 +95,10 @@ def needs_enqueue(backup, job_id=None):
     scheduled_jobs = scheduled.get_job_ids()
     started_jobs = started.get_job_ids()
 
-    if backup.device is None:
-        print(f'No device for {backup}')
-        return False
-    elif backup.status == StatusChoices.STATUS_DISABLED:
-        print(f'Backup disabled for {backup}')
-        return False
-    elif backup.device.status in [DeviceStatusChoices.STATUS_OFFLINE,
-                                DeviceStatusChoices.STATUS_FAILED,
-                                DeviceStatusChoices.STATUS_INVENTORY,
-                                DeviceStatusChoices.STATUS_PLANNED]:
-        print(f'Backup disabled for {backup} due to device status ({backup.device.status})')
-        return False
-    elif (backup.ip is None and backup.device.primary_ip is None) or backup.device.platform is None or \
-            backup.device.platform.napalm_driver == '' or backup.device.platform.napalm_driver is None:
-        print(f'Backup disabled for {backup} due to napalm drive or no primary IP ({backup.device.status})')
+    if not can_backup(backup):
         return False
     elif is_queued(backup, job_id):
-        print(f'Backup already queued for {backup}')
+        logger.info(f'Backup already queued for {backup}')
         return False
 
     return True