diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index e3cad73..0f0c142 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -31,7 +31,7 @@ jobs: strategy: max-parallel: 10 matrix: - netbox_version: ["v3.5.9", "v3.6.1"] + netbox_version: ["v3.5.9", "v3.6.4"] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/README.md b/README.md index 69b1c65..414600b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ ## About +### Collecting diffs + With this plugin you can find diff between the rendered configuration for a device to its actual configuration, retrieved from the device itself, or stored in DataSource. Read about [DataSources](https://demo.netbox.dev/static/docs/models/core/datasource/) for further details. @@ -23,6 +25,20 @@ Device configuration renders natively in NetBox. This [feature](https://demo.net NetBox Labs [blog](https://netboxlabs.com/blog/how-to-generate-device-configurations-with-netbox/) post about it. Plugin supports a wide list of vendors (Cisco, Juniper, Huawei, MicroTik etc.) with the help of Scrapli. Read [Scrapli](https://carlmontanari.github.io/scrapli/user_guide/project_details/#supported-platforms) and [scrapli-community](https://scrapli.github.io/scrapli_community/user_guide/project_details/#supported-platforms) documentations to find full list of vendors. + +### Pushing configuration + +Also you can push rendered configuration from NetBox to device and apply it. + +Supported platforms: + +* `arista_eos` +* `cisco_iosxe` +* `cisco_iosxr` +* `cisco_nxos` +* `juniper_junos` + +This is possible thanks to the scrapli_cfg. Read [Scrapli](https://github.com/scrapli/scrapli_cfg/) documentation for more info. ## Compatibility diff --git a/docs/changelog.md b/docs/changelog.md index fcf94fb..d0145fb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## 2.0.0 (2023-10-XX) + +* [#25](https://github.com/miaow2/netbox-config-diff/issues/25) Add configuration management + ## 1.2.2 (2023-09-29) * [#28](https://github.com/miaow2/netbox-config-diff/issues/28) Add legacy ssh algorithms to support old OS versions diff --git a/docs/usage.md b/docs/colliecting-diffs.md similarity index 98% rename from docs/usage.md rename to docs/colliecting-diffs.md index b4da647..35d3057 100644 --- a/docs/usage.md +++ b/docs/colliecting-diffs.md @@ -32,7 +32,7 @@ In the script, you can define a site, on which devices run compliance, or device If you define both fields, script will run only on devices from `Devices` field !!! warning - Script runs only on devices with status `Active`, assigned Primary IP, Platform and PlatformSetting + Script runs only on devices with status `Active`, assigned Primary IP, Config Template, Platform and PlatformSetting If you have configs in NetBox DataSource, you can define it, the script instead of connecting to devices will find configs in DataSource by device's names. diff --git a/docs/configuratiom-management.md b/docs/configuratiom-management.md new file mode 100644 index 0000000..c95c0aa --- /dev/null +++ b/docs/configuratiom-management.md @@ -0,0 +1,114 @@ +# Usage + +With plugin you can push rendered configuration from NetBox to devices. + +Supported platforms: + +* `arista_eos` +* `cisco_iosxe` +* `cisco_iosxr` +* `cisco_nxos` +* `juniper_junos` + +Plugin using [scrapli-cfg](https://github.com/scrapli/scrapli_cfg) for this feature. + +## Substitutes + +If you render not full configuration, it is acceptable to pull missing config sections from the actual configuration to render full configuration. + +!!! note + If you render full configuration in NetBox, you can proceed to `Configuration Request` part + +To do that you should create substitute. + +Substitutes is a "tag" that needs to be replaced with output from the real device, and a regex pattern that "pulls" this section from the actual device itself. + +![Screenshot of the substitute](media/screenshots/substitute.png) + +In screenshot below we add substitute for Arista PlatformSetting + +* **Name** is a "tag", you should put this as jinja2 variable in your config template in NetBox +* **Regexp** is a regex, that "pulls" what matched from device and replace `Name` jinja2 variable in config template + +In example substitute `ethernet_interfaces` section will be replaced with whatever the provided pattern finds from the real device. + +This pattern matches all ethernet interfaces on a Arista device. + +To correctly render substitute in config template you have two options: + +``` +{{ "{{ ethernet_interfaces }}" }} +``` + +or + +``` +{% raw %}{{ ethernet_interfaces }}{% endraw %} +``` + +Config template will look like: + +![Screenshot of the config template with substitute](media/screenshots/config-temp-substitute.png) + +And rendered config template with substitute + +![Screenshot of the rendered template with substitute](media/screenshots/render-temp-substitute.png) + +## Configuration Request + +Now you let's create `Configuration Request` with devices you want to configure. + +!!! warning + For request only accepts devices with `Active` status and assigned Platform, Primary IP, Config Template and PlatformSetting + +Find `Configuration Requests` in navbar. + +Now collect diffs for devices pressing `Collecting diffs` button. + +![Screenshot of the Collecting diffs button](media/screenshots/cr-collecting-diff-button.png) + +On tab `Diffs` you can review diffs for devices. + +![Screenshot of the Diffs tab](media/screenshots/cr-diffs-tab.png) + +To continue approve request by pressing `Approve` button. + +![Screenshot of the Approve button](media/screenshots/cr-approve-button.png) + +Also you can cancel approve after that. + +![Screenshot of the Unapprove button](media/screenshots/cr-unapprove-button.png) + +After approval you can see by whom configuration request is approved. + +![Screenshot of the Approved request](media/screenshots/cr-approved.png) + +At this moment you can schedule job that will push rendered configuration to devices in configuration request by pressing schedule button. + +![Screenshot of the Schedule button](media/screenshots/cr-schedule-button.png) + +After that you can see by whom configuration request is scheduled and time. + +![Screenshot of the Scheduled request](media/screenshots/cr-scheduled.png) + +Also you can cancel scheduled job by pressing `Unschedule` button. + +![Screenshot of the Unschedule button](media/screenshots/cr-unschedule-button.png) + +!!! warning + Approve and Schedule buttons is accessable only to user with `netbox_config_diff.approve_configurationrequest` + permission + +!!! warning + If you unapprove scheduled configuration request, scheduled job will be canceled + +After scheduled job is completed you can job logs on configuration request page. + +![Screenshot of the Unschedule button](media/screenshots/cr-job-log.png) + +!!! note + Completed configuration requests can't be edited. + +## Rollback + +If an error occurs while executing a job that pushes configurations to devices then all configured devices will be rollbacked to the previous version of the configuration. diff --git a/docs/index.md b/docs/index.md index 522f8d6..319426c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,4 +12,4 @@ ## Usage -Read this [doc](usage.md) to find out how to use plugin +Read this [doc](colliecting-diffs.md) about collecting diffs, for configuration management read [this](configuratiom-management.md) diff --git a/docs/media/screenshots/config-temp-substitute.png b/docs/media/screenshots/config-temp-substitute.png new file mode 100644 index 0000000..700aba6 Binary files /dev/null and b/docs/media/screenshots/config-temp-substitute.png differ diff --git a/docs/media/screenshots/cr-approve-button.png b/docs/media/screenshots/cr-approve-button.png new file mode 100644 index 0000000..99fd64d Binary files /dev/null and b/docs/media/screenshots/cr-approve-button.png differ diff --git a/docs/media/screenshots/cr-approved.png b/docs/media/screenshots/cr-approved.png new file mode 100644 index 0000000..d545f7a Binary files /dev/null and b/docs/media/screenshots/cr-approved.png differ diff --git a/docs/media/screenshots/cr-collecting-diff-button.png b/docs/media/screenshots/cr-collecting-diff-button.png new file mode 100644 index 0000000..1fdf224 Binary files /dev/null and b/docs/media/screenshots/cr-collecting-diff-button.png differ diff --git a/docs/media/screenshots/cr-diffs-tab.png b/docs/media/screenshots/cr-diffs-tab.png new file mode 100644 index 0000000..4134a86 Binary files /dev/null and b/docs/media/screenshots/cr-diffs-tab.png differ diff --git a/docs/media/screenshots/cr-job-log.png b/docs/media/screenshots/cr-job-log.png new file mode 100644 index 0000000..6df5508 Binary files /dev/null and b/docs/media/screenshots/cr-job-log.png differ diff --git a/docs/media/screenshots/cr-schedule-button.png b/docs/media/screenshots/cr-schedule-button.png new file mode 100644 index 0000000..7baf7cc Binary files /dev/null and b/docs/media/screenshots/cr-schedule-button.png differ diff --git a/docs/media/screenshots/cr-scheduled.png b/docs/media/screenshots/cr-scheduled.png new file mode 100644 index 0000000..a61304b Binary files /dev/null and b/docs/media/screenshots/cr-scheduled.png differ diff --git a/docs/media/screenshots/cr-unapprove-button.png b/docs/media/screenshots/cr-unapprove-button.png new file mode 100644 index 0000000..7a0ef97 Binary files /dev/null and b/docs/media/screenshots/cr-unapprove-button.png differ diff --git a/docs/media/screenshots/cr-unschedule-button.png b/docs/media/screenshots/cr-unschedule-button.png new file mode 100644 index 0000000..aaf0c90 Binary files /dev/null and b/docs/media/screenshots/cr-unschedule-button.png differ diff --git a/docs/media/screenshots/navbar.png b/docs/media/screenshots/navbar.png index 02e910f..167889f 100644 Binary files a/docs/media/screenshots/navbar.png and b/docs/media/screenshots/navbar.png differ diff --git a/docs/media/screenshots/render-temp-substitute.png b/docs/media/screenshots/render-temp-substitute.png new file mode 100644 index 0000000..d3ed33f Binary files /dev/null and b/docs/media/screenshots/render-temp-substitute.png differ diff --git a/docs/media/screenshots/substitute.png b/docs/media/screenshots/substitute.png new file mode 100644 index 0000000..0a9a228 Binary files /dev/null and b/docs/media/screenshots/substitute.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 9f4a0d9..a6a6e94 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,7 +6,8 @@ repo_name: miaow2/netbox-config-diff nav: - Home: index.md - User Guide: - - Quick Start Guide: usage.md + - Collecting diffs: colliecting-diffs.md + - Configuration management: configuratiom-management.md - Integration with secrets: secrets.md - Screenshots: screenshots.md - Contributing: contributing.md diff --git a/netbox_config_diff/api/serializers.py b/netbox_config_diff/api/serializers.py index 26195cd..bf38323 100644 --- a/netbox_config_diff/api/serializers.py +++ b/netbox_config_diff/api/serializers.py @@ -1,10 +1,15 @@ from dcim.api.serializers import NestedDeviceSerializer, NestedPlatformSerializer -from netbox.api.fields import ChoiceField +from dcim.models import Device +from netbox.api.fields import ChoiceField, SerializedPKRelatedField from netbox.api.serializers import NetBoxModelSerializer from rest_framework import serializers +from rest_framework.serializers import ValidationError +from users.api.nested_serializers import NestedUserSerializer +from utilities.utils import local_now -from netbox_config_diff.choices import ConfigComplianceStatusChoices -from netbox_config_diff.models import ConfigCompliance, PlatformSetting +from netbox_config_diff.choices import ConfigComplianceStatusChoices, ConfigurationRequestStatusChoices +from netbox_config_diff.constants import ACCEPTABLE_DRIVERS +from netbox_config_diff.models import ConfigCompliance, ConfigurationRequest, PlatformSetting, Substitute class ConfigComplianceSerializer(NetBoxModelSerializer): @@ -45,6 +50,113 @@ class Meta: "driver", "command", "exclude_regex", + "description", + "tags", + "custom_fields", + "created", + "last_updated", + ) + + +class NestedPlatformSettingSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name="plugins-api:netbox_config_diff-api:platformsetting-detail") + + class Meta: + model = PlatformSetting + fields = ("id", "url", "display", "driver") + + +class ConfigurationRequestSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField( + view_name="plugins-api:netbox_config_diff-api:configurationrequest-detail" + ) + devices = SerializedPKRelatedField( + queryset=Device.objects.all(), + serializer=NestedDeviceSerializer, + many=True, + ) + status = ChoiceField(choices=ConfigurationRequestStatusChoices, read_only=True) + created_by = NestedUserSerializer(read_only=True) + approved_by = NestedUserSerializer(read_only=True) + scheduled_by = NestedUserSerializer(read_only=True) + + class Meta: + model = ConfigurationRequest + fields = ( + "id", + "url", + "display", + "devices", + "created_by", + "approved_by", + "scheduled_by", + "scheduled", + "started", + "completed", + "status", + "description", + "comments", + "tags", + "custom_fields", + "created", + "last_updated", + ) + read_only_fields = ["started", "scheduled", "completed"] + + def validate(self, data): + if data.get("devices"): + if devices := data["devices"].filter(platform__platform_setting__isnull=True): + platforms = {d.platform.name for d in devices} + raise ValidationError({"devices": f"Assign PlatformSetting for platform(s): {', '.join(platforms)}"}) + + if drivers := { + device.platform.platform_setting.driver + for device in data["devices"] + if device.platform.platform_setting.driver not in ACCEPTABLE_DRIVERS + }: + raise ValidationError({"devices": f"Driver(s) not supported: {', '.join(drivers)}"}) + + return super().validate(data) + + +class ConfigurationRequestRWSerializer(ConfigurationRequestSerializer): + created_by = NestedUserSerializer() + + +class NestedConfigurationRequestSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField( + view_name="plugins-api:netbox_config_diff-api:configurationrequest-detail" + ) + status = ChoiceField(choices=ConfigurationRequestStatusChoices) + + class Meta: + model = ConfigurationRequest + fields = ("id", "url", "display", "status") + + +class ConfigurationRequestScheduleSerializer(serializers.Serializer): + schedule_at = serializers.DateTimeField() + + def validate_schedule_at(self, value): + if value < local_now(): + raise serializers.ValidationError("Scheduled time must be in the future.") + return value + + +class SubstituteSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name="plugins-api:netbox_config_diff-api:substitute-detail") + platform_setting = NestedPlatformSettingSerializer() + + class Meta: + model = Substitute + fields = ( + "id", + "url", + "display", + "platform_setting", + "name", + "description", + "regexp", "tags", "custom_fields", "created", diff --git a/netbox_config_diff/api/urls.py b/netbox_config_diff/api/urls.py index d79a2b3..9a1863d 100644 --- a/netbox_config_diff/api/urls.py +++ b/netbox_config_diff/api/urls.py @@ -7,5 +7,7 @@ router = NetBoxRouter() router.register("config-compliances", views.ConfigComplianceViewSet) router.register("platform-settings", views.PlatformSettingViewSet) +router.register("configuration-requests", views.ConfigurationRequestViewSet) +router.register("substitutes", views.SubstituteViewSet) urlpatterns = router.urls diff --git a/netbox_config_diff/api/views.py b/netbox_config_diff/api/views.py index cbb2df9..38f9674 100644 --- a/netbox_config_diff/api/views.py +++ b/netbox_config_diff/api/views.py @@ -1,9 +1,35 @@ +from core.api.serializers import JobSerializer +from core.choices import JobStatusChoices +from django.contrib.auth import get_user_model +from django.shortcuts import get_object_or_404 +from django_rq.queues import get_connection, get_queue from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet +from netbox.constants import RQ_QUEUE_DEFAULT +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied +from rest_framework.response import Response +from rq import Worker +from rq.exceptions import InvalidJobOperation +from utilities.exceptions import RQWorkerNotRunningException -from netbox_config_diff.filtersets import ConfigComplianceFilterSet, PlatformSettingFilterSet -from netbox_config_diff.models import ConfigCompliance, PlatformSetting +from netbox_config_diff.choices import ConfigurationRequestStatusChoices +from netbox_config_diff.filtersets import ( + ConfigComplianceFilterSet, + ConfigurationRequestFilterSet, + PlatformSettingFilterSet, + SubstituteFilterSet, +) +from netbox_config_diff.models import ConfigCompliance, ConfigurationRequest, PlatformSetting, Substitute -from .serializers import ConfigComplianceSerializer, PlatformSettingSerializer +from .serializers import ( + ConfigComplianceSerializer, + ConfigurationRequestRWSerializer, + ConfigurationRequestScheduleSerializer, + ConfigurationRequestSerializer, + PlatformSettingSerializer, + SubstituteSerializer, +) class ConfigComplianceViewSet(NetBoxReadOnlyModelViewSet): @@ -16,3 +42,141 @@ class PlatformSettingViewSet(NetBoxModelViewSet): queryset = PlatformSetting.objects.prefetch_related("platform", "tags") serializer_class = PlatformSettingSerializer filterset_class = PlatformSettingFilterSet + + +class ConfigurationRequestViewSet(NetBoxModelViewSet): + queryset = ConfigurationRequest.objects.prefetch_related( + "devices", "created_by", "approved_by", "scheduled_by", "tags" + ) + serializer_class = ConfigurationRequestSerializer + filterset_class = ConfigurationRequestFilterSet + + def create(self, request, *args, **kwargs): + serializer = ConfigurationRequestRWSerializer( + data=request.data | {"created_by": request.user.pk}, context={"request": request} + ) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + @action(detail=True, methods=["post"]) + def approve(self, request, pk): + if not request.user.has_perm("netbox_config_diff.approve_configurationrequest"): + raise PermissionDenied( + "Approving configuration requests requires the " + "netbox_config_diff.approve_configurationrequest permission." + ) + + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + return Response( + {"description": f"{obj} is finished, you can't change it."}, status=status.HTTP_400_BAD_REQUEST + ) + + if obj.approved_by: + obj.approved_by = None + obj.status = ConfigurationRequestStatusChoices.CREATED + if obj.scheduled: + obj.scheduled = None + obj.scheduled_by = None + else: + obj.approved_by = get_user_model().objects.filter(pk=request.user.pk).first() + obj.status = ConfigurationRequestStatusChoices.APPROVED + obj.save() + + serializer = ConfigurationRequestSerializer(obj, context={"request": request}) + + return Response(serializer.data) + + @action(detail=True, methods=["post"]) + def schedule(self, request, pk): + if not request.user.has_perm("netbox_config_diff.approve_configurationrequest"): + raise PermissionDenied( + "Scheduling configuration requests requires the" + "netbox_config_diff.approve_configurationrequest permission." + ) + + if not Worker.count(get_connection(RQ_QUEUE_DEFAULT)): + raise RQWorkerNotRunningException() + + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + return Response( + {"description": f"{obj} is finished, you can't change it."}, status=status.HTTP_400_BAD_REQUEST + ) + if obj.scheduled: + return Response({"description": f"{obj} already scheduled"}, status=status.HTTP_400_BAD_REQUEST) + if obj.approved_by is None: + return Response({"description": f"Approve {obj} before schedule."}, status=status.HTTP_400_BAD_REQUEST) + + input_serializer = ConfigurationRequestScheduleSerializer(data=request.data) + if input_serializer.is_valid(): + obj.scheduled = input_serializer.validated_data.get("schedule_at") + obj.status = ConfigurationRequestStatusChoices.SCHEDULED + obj.scheduled_by = get_user_model().objects.filter(pk=request.user.pk).first() + obj.save() + serializer = ConfigurationRequestSerializer(obj, context={"request": request}) + obj.enqueue_job(request, "push_configs", schedule_at=input_serializer.validated_data.get("schedule_at")) + return Response(serializer.data) + + return Response(input_serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=True, methods=["post"]) + def unschedule(self, request, pk): + if not request.user.has_perm("netbox_config_diff.approve_configurationrequest"): + raise PermissionDenied( + "Scheduling configuration requests requires the" + "netbox_config_diff.approve_configurationrequest permission." + ) + + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + return Response( + {"description": f"{obj} is finished, you can't change it."}, status=status.HTTP_400_BAD_REQUEST + ) + if obj.approved_by is None: + return Response({"description": f"Approve {obj} before unschedule."}, status=status.HTTP_400_BAD_REQUEST) + + if obj.scheduled_by: + obj.scheduled = None + obj.scheduled_by = None + obj.status = ConfigurationRequestStatusChoices.APPROVED + obj.save() + queue = get_queue(RQ_QUEUE_DEFAULT) + for result in obj.jobs.filter(name__contains="push_configs", status=JobStatusChoices.STATUS_SCHEDULED): + result.delete() + if job := queue.fetch_job(str(result.job_id)): + try: + job.cancel() + except InvalidJobOperation: + pass + + serializer = ConfigurationRequestSerializer(obj, context={"request": request}) + return Response(serializer.data) + + @action(detail=True, methods=["post"], url_path="collect-diffs") + def collect_diffs(self, request, pk): + if not request.user.has_perm("netbox_config_diff.change_configurationrequest"): + raise PermissionDenied( + "Collecting diffs requires the netbox_config_diff.change_configurationrequest permission." + ) + + if not Worker.count(get_connection(RQ_QUEUE_DEFAULT)): + raise RQWorkerNotRunningException() + + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + return Response( + {"description": f"{obj} is finished, you can't change it."}, status=status.HTTP_400_BAD_REQUEST + ) + + job = obj.enqueue_job(request, "collect_diffs") + serializer = JobSerializer(job, context={"request": request}) + return Response(serializer.data) + + +class SubstituteViewSet(NetBoxModelViewSet): + queryset = Substitute.objects.prefetch_related("platform_setting", "tags") + serializer_class = SubstituteSerializer + filterset_class = SubstituteFilterSet diff --git a/netbox_config_diff/choices.py b/netbox_config_diff/choices.py index e1e443b..3fb5714 100644 --- a/netbox_config_diff/choices.py +++ b/netbox_config_diff/choices.py @@ -6,10 +6,38 @@ class ConfigComplianceStatusChoices(ChoiceSet): PENDING = "pending" FAILED = "failed" ERRORED = "errored" + DIFF = "diff" CHOICES = ( (COMPLIANT, "Compliant", "green"), (PENDING, "Pending", "cyan"), (FAILED, "Failed", "red"), (ERRORED, "Errored", "red"), + (DIFF, "Diff", "teal"), + ) + + +class ConfigurationRequestStatusChoices(ChoiceSet): + CREATED = "created" + APPROVED = "approved" + SCHEDULED = "scheduled" + RUNNING = "running" + FAILED = "failed" + ERRORED = "errored" + COMPLETED = "completed" + + CHOICES = ( + (CREATED, "Created", "cyan"), + (APPROVED, "Approved", "indigo"), + (SCHEDULED, "Scheduled", "teal"), + (RUNNING, "Running", "blue"), + (FAILED, "Failed", "red"), + (ERRORED, "Errored", "red"), + (COMPLETED, "Completed", "green"), + ) + + FINISHED_STATE_CHOICES = ( + FAILED, + ERRORED, + COMPLETED, ) diff --git a/netbox_config_diff/compliance/base.py b/netbox_config_diff/compliance/base.py index bf8c64d..f902019 100644 --- a/netbox_config_diff/compliance/base.py +++ b/netbox_config_diff/compliance/base.py @@ -1,4 +1,5 @@ import asyncio +import re import traceback from typing import Iterable, Iterator @@ -19,17 +20,13 @@ from .secrets import SecretsMixin from .utils import PLATFORM_MAPPING, exclude_lines, get_unified_diff -try: - from extras.plugins import get_installed_plugins, get_plugin_config -except ImportError: - from extras.plugins.utils import get_installed_plugins, get_plugin_config - class ConfigDiffBase(SecretsMixin): site = ObjectVar( model=Site, required=False, - description="Run compliance for devices (with status Active, primary IP and platform) in this site", + description="Run compliance for devices (with status Active, " + "primary IP, platform and config template) in this site", ) devices = MultiObjectVar( model=Device, @@ -38,6 +35,7 @@ class ConfigDiffBase(SecretsMixin): "status": DeviceStatusChoices.STATUS_ACTIVE, "has_primary_ip": True, "platform_id__n": "null", + "config_template_id__n": "null", }, description="If you define devices in this field, the Site field will be ignored", ) @@ -70,6 +68,7 @@ def validate_data(self, data: dict) -> Iterable[Device]: .filter( status=DeviceStatusChoices.STATUS_ACTIVE, platform__platform_setting__isnull=False, + config_template__isnull=False, ) .exclude( Q(primary_ip4__isnull=True) & Q(primary_ip6__isnull=True), @@ -80,6 +79,7 @@ def validate_data(self, data: dict) -> Iterable[Device]: site=data["site"], status=DeviceStatusChoices.STATUS_ACTIVE, platform__platform_setting__isnull=False, + config_template__isnull=False, ).exclude( Q(primary_ip4__isnull=True) & Q(primary_ip6__isnull=True), ) @@ -116,13 +116,10 @@ def log_results(self, device: DeviceDataClass) -> None: self.log_success(f"{device.name} no diff") def get_devices_with_rendered_configs(self, devices: Iterable[Device]) -> Iterator[DeviceDataClass]: - if "netbox_secrets" in get_installed_plugins(): - self.get_master_key() - self.user_role = get_plugin_config("netbox_config_diff", "USER_SECRET_ROLE") - self.password_role = get_plugin_config("netbox_config_diff", "PASSWORD_SECRET_ROLE") + self.check_netbox_secrets() + self.substitutes = {} for device in devices: username, password = self.get_credentials(device) - self.log_info(f"{username} {password}") rendered_config = None error = None context_data = device.get_config_context() @@ -130,16 +127,22 @@ def get_devices_with_rendered_configs(self, devices: Iterable[Device]) -> Iterat if config_template := device.get_config_template(): try: rendered_config = config_template.render(context=context_data) + rendered_config = re.sub(r"{{.+}}\s+", "", rendered_config) except TemplateError: error = traceback.format_exc() else: error = "Define config template for device" + platform = device.platform.platform_setting.driver + if not self.substitutes.get(platform): + if substitutes := device.platform.platform_setting.substitutes.all(): + self.substitutes[platform] = [s.regexp for s in substitutes] + yield DeviceDataClass( pk=device.pk, name=device.name, mgmt_ip=str(device.primary_ip.address.ip), - platform=device.platform.platform_setting.driver, + platform=platform, command=device.platform.platform_setting.command, exclude_regex=device.platform.platform_setting.exclude_regex, username=username, @@ -169,7 +172,9 @@ def get_diff(self, devices: list[DeviceDataClass]) -> None: for device in devices: if device.error is not None: continue - cleaned_config = exclude_lines(device.actual_config, device.exclude_regex) + cleaned_config = exclude_lines(device.actual_config, device.exclude_regex.splitlines()) + if self.substitutes.get(device.platform): + cleaned_config = exclude_lines(cleaned_config, self.substitutes[device.platform]) device.diff = get_unified_diff(device.rendered_config, cleaned_config, device.name) if device.platform in PLATFORM_MAPPING: device.missing = diff_network_config( @@ -178,16 +183,3 @@ def get_diff(self, devices: list[DeviceDataClass]) -> None: device.extra = diff_network_config( cleaned_config, device.rendered_config, PLATFORM_MAPPING[device.platform] ) - - def get_credentials(self, device: Device) -> tuple[str, str]: - username = get_plugin_config("netbox_config_diff", "USERNAME") - password = get_plugin_config("netbox_config_diff", "PASSWORD") - if "netbox_secrets" in get_installed_plugins(): - if secret := device.secrets.filter(role__name=self.user_role).first(): - if value := self.get_secret(secret): - username = value - if secret := device.secrets.filter(role__name=self.password_role).first(): - if value := self.get_secret(secret): - password = value - - return username, password diff --git a/netbox_config_diff/compliance/models.py b/netbox_config_diff/compliance/models.py index 4f135b9..f16077e 100644 --- a/netbox_config_diff/compliance/models.py +++ b/netbox_config_diff/compliance/models.py @@ -12,9 +12,9 @@ class DeviceDataClass: name: str mgmt_ip: str platform: str - command: str username: str password: str + command: str | None = None exclude_regex: str | None = None rendered_config: str | None = None actual_config: str | None = None @@ -22,12 +22,16 @@ class DeviceDataClass: missing: str | None = None extra: str | None = None error: str | None = None + config_error: str | None = None auth_strict_key: bool = False transport: str = "asyncssh" def __str__(self): return self.name + def __hash__(self): + return hash(self.name) + def to_scrapli(self): return { "host": self.mgmt_ip, @@ -78,7 +82,7 @@ def to_db(self): if self.error: status = ConfigComplianceStatusChoices.ERRORED elif self.diff: - status = ConfigComplianceStatusChoices.FAILED + status = ConfigComplianceStatusChoices.DIFF else: status = ConfigComplianceStatusChoices.COMPLIANT diff --git a/netbox_config_diff/compliance/secrets.py b/netbox_config_diff/compliance/secrets.py index 85af298..998311c 100644 --- a/netbox_config_diff/compliance/secrets.py +++ b/netbox_config_diff/compliance/secrets.py @@ -1,11 +1,18 @@ import base64 from typing import TYPE_CHECKING +from dcim.models import Device +from extras.plugins import get_installed_plugins, get_plugin_config + if TYPE_CHECKING: from netbox_secrets.models import Secret class SecretsMixin: + username: str + password: str + netbox_secrets_installed: bool = False + def get_session_key(self) -> None: if "netbox_secrets_sessionid" in self.request.COOKIES: self.session_key = base64.b64decode(self.request.COOKIES['netbox_secrets_sessionid']) @@ -26,7 +33,10 @@ def get_master_key(self) -> None: sk = SessionKey.objects.get(userkey__user=self.request.user) self.master_key = sk.get_master_key(self.session_key) except Exception as e: - self.log_failure(f"Can't fetch master_key: {str(e)}") + if getattr(self, "logger"): + self.logger.log_failure(f"Can't fetch master_key: {str(e)}") + else: + self.log_failure(f"Can't fetch master_key: {str(e)}") def get_secret(self, secret: "Secret") -> str | None: try: @@ -34,3 +44,25 @@ def get_secret(self, secret: "Secret") -> str | None: except Exception: return None return secret.plaintext + + def get_credentials(self, device: Device) -> tuple[str, str]: + if self.netbox_secrets_installed: + if secret := device.secrets.filter(role__name=self.user_role).first(): + if value := self.get_secret(secret): + username = value + if secret := device.secrets.filter(role__name=self.password_role).first(): + if value := self.get_secret(secret): + password = value + return username, password + + return self.username, self.password + + def check_netbox_secrets(self) -> None: + if "netbox_secrets" in get_installed_plugins(): + self.get_master_key() + self.user_role = get_plugin_config("netbox_config_diff", "USER_SECRET_ROLE") + self.password_role = get_plugin_config("netbox_config_diff", "PASSWORD_SECRET_ROLE") + self.netbox_secrets_installed = True + else: + self.username = get_plugin_config("netbox_config_diff", "USERNAME") + self.password = get_plugin_config("netbox_config_diff", "PASSWORD") diff --git a/netbox_config_diff/compliance/utils.py b/netbox_config_diff/compliance/utils.py index f714780..d3729d5 100644 --- a/netbox_config_diff/compliance/utils.py +++ b/netbox_config_diff/compliance/utils.py @@ -28,7 +28,7 @@ def get_unified_diff(rendered_config: str, actual_config: str, device: str) -> s return "\n".join(diff).strip() -def exclude_lines(text: str, regex: str) -> str: - for item in regex.splitlines(): - text = re.sub(item, "", text, flags=re.MULTILINE) +def exclude_lines(text: str, regexs: list) -> str: + for item in regexs: + text = re.sub(item, "", text, flags=re.I | re.M) return text.strip() diff --git a/netbox_config_diff/configurator/__init__.py b/netbox_config_diff/configurator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/netbox_config_diff/configurator/base.py b/netbox_config_diff/configurator/base.py new file mode 100644 index 0000000..1e546a7 --- /dev/null +++ b/netbox_config_diff/configurator/base.py @@ -0,0 +1,211 @@ +import asyncio +import re +import traceback +from contextlib import asynccontextmanager +from typing import AsyncIterator, Iterable + +from asgiref.sync import sync_to_async +from dcim.models import Device +from jinja2.exceptions import TemplateError +from netutils.config.compliance import diff_network_config +from scrapli import AsyncScrapli +from scrapli_cfg.platform.base.async_platform import AsyncScrapliCfgPlatform +from scrapli_cfg.response import ScrapliCfgResponse +from utilities.utils import NetBoxFakeRequest + +from netbox_config_diff.compliance.models import DeviceDataClass +from netbox_config_diff.compliance.secrets import SecretsMixin +from netbox_config_diff.compliance.utils import PLATFORM_MAPPING, get_unified_diff +from netbox_config_diff.configurator.exceptions import DeviceConfigurationError, DeviceValidationError +from netbox_config_diff.configurator.utils import CustomLogger +from netbox_config_diff.constants import ACCEPTABLE_DRIVERS +from netbox_config_diff.models import ConfigCompliance + +from .factory import AsyncScrapliCfg + + +class Configurator(SecretsMixin): + def __init__(self, devices: Iterable[Device], request: NetBoxFakeRequest) -> None: + self.devices = devices + self.request = request + self.unprocessed_devices = set() + self.processed_devices = set() + self.failed_devices = set() + self.substitutes: dict[str, list] = {} + self.logger = CustomLogger() + self.connections: dict[str, AsyncScrapliCfgPlatform] = {} + + def validate_devices(self) -> None: + self.check_netbox_secrets() + for device in self.devices: + username, password = self.get_credentials(device) + if device.platform.platform_setting is None: + self.logger.log_warning(f"Skipping {device}, add PlatformSetting for {device.platform} platform") + elif device.platform.platform_setting.driver not in ACCEPTABLE_DRIVERS: + self.logger.log_warning( + f"Skipping {device}, driver {device.platform.platform_setting.driver} is not supported" + ) + else: + rendered_config = None + error = None + context_data = device.get_config_context() + context_data.update({"device": device}) + if config_template := device.get_config_template(): + try: + rendered_config = config_template.render(context=context_data) + except TemplateError: + error = traceback.format_exc() + self.logger.log_failure(error) + else: + error = "Define config template for device" + self.logger.log_failure(error) + + d = DeviceDataClass( + pk=device.pk, + name=device.name, + mgmt_ip=str(device.primary_ip.address.ip), + platform=device.platform.platform_setting.driver, + username=username, + password=password, + rendered_config=rendered_config, + error=error, + ) + if error: + self.failed_devices.add(d) + else: + self.connections[d.name] = AsyncScrapliCfg( + conn=AsyncScrapli(**d.to_scrapli()), dedicated_connection=True + ) + self.unprocessed_devices.add(d) + if not self.substitutes.get(d.platform): + if substitutes := device.platform.platform_setting.substitutes.all(): + self.substitutes[d.platform] = [ + (s.name, re.compile(s.regexp, flags=re.I | re.M)) for s in substitutes + ] + + if self.failed_devices: + raise DeviceValidationError( + "Error in validating devices", devices=", ".join(f"{d.name}: {d.error}" for d in self.failed_devices) + ) + + self.logger.log_info(f"Working with {', '.join(d.name for d in self.unprocessed_devices)}") + + @asynccontextmanager + async def connection(self, only_processed_devices: bool = False) -> AsyncIterator[None]: + if only_processed_devices: + connections = [self.connections[device.name] for device in self.processed_devices] + else: + connections = self.connections.values() + try: + await asyncio.gather(*(conn.__aenter__() for conn in connections)) + yield + finally: + await asyncio.gather(*(conn.__aexit__(None, None, None) for conn in connections)) + + def collect_diffs(self) -> None: + loop = asyncio.get_event_loop() + loop.run_until_complete(self._collect_diffs()) + + @sync_to_async + def update_diffs(self) -> None: + for device in self.unprocessed_devices: + try: + obj = ConfigCompliance.objects.get(device_id=device.pk) + obj.snapshot() + obj.update(**device.to_db()) + obj.save() + except ConfigCompliance.DoesNotExist: + ConfigCompliance.objects.create(**device.to_db()) + + async def _collect_diffs(self) -> None: + async with self.connection(): + await asyncio.gather(*(self._collect_one_diff(d) for d in self.unprocessed_devices)) + await self.update_diffs() + + async def _collect_one_diff(self, device: DeviceDataClass) -> None: + self.logger.log_info(f"Collecting diff on {device.name}") + try: + conn = self.connections[device.name] + if substitutes := self.substitutes.get(device.platform): + actual_config, rendered_config = await conn.render_substituted_config( + config_template=device.rendered_config, substitutes=substitutes + ) + device.rendered_config = rendered_config + else: + actual_config = await conn.get_config() + device.actual_config = conn.clean_config(actual_config.result) + + device.diff = get_unified_diff(device.rendered_config, device.actual_config, device.name) + self.logger.add_diff(device.name, diff=device.diff) + device.missing = diff_network_config( + device.rendered_config, device.actual_config, PLATFORM_MAPPING[device.platform] + ) + device.extra = diff_network_config( + device.actual_config, device.rendered_config, PLATFORM_MAPPING[device.platform] + ) + self.logger.log_info(f"Got diff from {device.name}") + except Exception: + error = traceback.format_exc() + device.error = error + self.logger.log_failure(error) + self.logger.add_diff(device.name, error=error) + + def push_configs(self): + loop = asyncio.get_event_loop() + loop.run_until_complete(self._push_configs()) + + async def _push_configs(self) -> None: + async with self.connection(): + await asyncio.gather(*(self._collect_one_diff(d) for d in self.unprocessed_devices)) + await self.update_diffs() + await asyncio.gather(*(self._push_one_config(d) for d in self.unprocessed_devices)) + if self.failed_devices: + self.logger.log_warning(f"Failed device(s): {', '.join(d.name for d in self.failed_devices)}") + async with self.connection(only_processed_devices=True): + await self.rollback() + raise DeviceConfigurationError( + "Error in configuring devices", + devices=", ".join(f"{d.name}: {d.config_error}" for d in self.failed_devices), + ) + + async def _push_one_config(self, device: DeviceDataClass) -> None: + self.logger.log_info(f"Push config to {device.name}") + try: + conn = self.connections[device.name] + response = await conn.load_config(config=device.rendered_config, replace=True) + if response.failed: + await self.abort_config("load", conn, response, device.name) + return + response = await conn.commit_config() + if response.failed: + await self.abort_config("commit", conn, response, device.name) + return + self.unprocessed_devices.remove(device) + self.processed_devices.add(device) + self.logger.log_info(f"Successfully pushed config to {device.name}") + except Exception: + error = traceback.format_exc() + device.config_error = error + self.logger.log_failure(error) + self.unprocessed_devices.remove(device) + self.failed_devices.add(device) + + async def abort_config( + self, operation: str, conn: AsyncScrapliCfgPlatform, response: ScrapliCfgResponse, device: DeviceDataClass + ) -> None: + self.logger.log_failure(f"Failed to {operation} config on {device.name}: {response.result}") + device.config_error = response.result + await conn.abort_config() + self.unprocessed_devices.remove(device) + self.failed_devices.add(device) + self.logger.log_info(f"Aborted config on {device.name}") + + async def rollback(self) -> None: + self.logger.log_info(f"Rollback config: {', '.join(d.name for d in self.processed_devices)}") + await asyncio.gather(*(self._rollback_one(d) for d in self.processed_devices)) + + async def _rollback_one(self, device: DeviceDataClass) -> None: + conn = self.connections[device.name] + await conn.load_config(config=device.actual_config, replace=True) + await conn.commit_config() + self.logger.log_info(f"Successfully rollbacked {device.name}") diff --git a/netbox_config_diff/configurator/exceptions.py b/netbox_config_diff/configurator/exceptions.py new file mode 100644 index 0000000..acdce96 --- /dev/null +++ b/netbox_config_diff/configurator/exceptions.py @@ -0,0 +1,19 @@ +from typing import Any + + +class DeviceError(Exception): + def __str__(self) -> str: + return f"{self.message}, {self.kwargs['devices']}" + + def __init__(self, message: str, **kwargs: Any) -> None: + self.message = message + self.kwargs = kwargs + super().__init__(message) + + +class DeviceValidationError(DeviceError): + pass + + +class DeviceConfigurationError(DeviceError): + pass diff --git a/netbox_config_diff/configurator/factory.py b/netbox_config_diff/configurator/factory.py new file mode 100644 index 0000000..4ce153e --- /dev/null +++ b/netbox_config_diff/configurator/factory.py @@ -0,0 +1,99 @@ +from typing import TYPE_CHECKING, Any, Callable + +from scrapli.driver.core import ( + AsyncEOSDriver, + AsyncIOSXEDriver, + AsyncIOSXRDriver, + AsyncJunosDriver, + AsyncNXOSDriver, +) +from scrapli.driver.network import AsyncNetworkDriver, NetworkDriver +from scrapli_cfg.exceptions import ScrapliCfgException +from scrapli_cfg.logging import logger + +from .platforms import ( + CustomAsyncScrapliCfgEOS, + CustomAsyncScrapliCfgIOSXE, + CustomAsyncScrapliCfgIOSXR, + CustomAsyncScrapliCfgJunos, + CustomAsyncScrapliCfgNXOS, +) + +ASYNC_CORE_PLATFORM_MAP = { + AsyncEOSDriver: CustomAsyncScrapliCfgEOS, + AsyncIOSXEDriver: CustomAsyncScrapliCfgIOSXE, + AsyncIOSXRDriver: CustomAsyncScrapliCfgIOSXR, + AsyncNXOSDriver: CustomAsyncScrapliCfgNXOS, + AsyncJunosDriver: CustomAsyncScrapliCfgJunos, +} + +if TYPE_CHECKING: + from scrapli_cfg.platform.base.async_platform import AsyncScrapliCfgPlatform + + +def AsyncScrapliCfg( + conn: AsyncNetworkDriver, + *, + config_sources: list[str] | None = None, + on_prepare: Callable[..., Any] | None = None, + dedicated_connection: bool = False, + ignore_version: bool = False, + **kwargs: Any, +) -> "AsyncScrapliCfgPlatform": + """ + Scrapli Config Async Factory + + Return a async scrapli config object for the provided platform. Prefer to use factory classes + just so that the naming convention (w/ upper case things) is "right", but given that the class + version inherited from the base ScrapliCfgPlatform and did not implement the abstract methods + this felt like a better move. + + Args: + conn: scrapli connection to use + config_sources: list of config sources + on_prepare: optional callable to run at connection `prepare` + dedicated_connection: if `False` (default value) scrapli cfg will not open or close the + underlying scrapli connection and will raise an exception if the scrapli connection + is not open. If `True` will automatically open and close the scrapli connection when + using with a context manager, `prepare` will open the scrapli connection (if not + already open), and `close` will close the scrapli connection. + ignore_version: ignore checking device version support; currently this just means that + scrapli-cfg will not fetch the device version during the prepare phase, however this + will (hopefully) be used in the future to limit what methods can be used against a + target device. For example, for EOS devices we need > 4.14 to load configs; so if a + device is encountered at 4.13 the version check would raise an exception rather than + just failing in a potentially awkward fashion. + kwargs: keyword args to pass to the scrapli_cfg object (for things like iosxe 'filesystem' + argument) + + Returns: + AsyncScrapliCfg: async scrapli cfg object + + Raises: + ScrapliCfgException: if provided connection object is sync + ScrapliCfgException: if provided connection object is async but is not a supported ("core") + platform type + + """ + logger.debug("AsyncScrapliCfg factory initialized") + + if isinstance(conn, NetworkDriver): + raise ScrapliCfgException( + "provided scrapli connection is sync but using 'AsyncScrapliCfg' -- you must use an " + "async connection with 'AsyncScrapliCfg'!" + ) + + platform_class = ASYNC_CORE_PLATFORM_MAP.get(type(conn)) + if not platform_class: + raise ScrapliCfgException(f"scrapli connection object type '{type(conn)}' not a supported scrapli-cfg type") + + final_platform: "AsyncScrapliCfgPlatform" = platform_class( + conn=conn, + config_sources=config_sources, + on_prepare=on_prepare, + dedicated_connection=dedicated_connection, + ignore_version=ignore_version, + **kwargs, + ) + + return final_platform diff --git a/netbox_config_diff/configurator/platforms.py b/netbox_config_diff/configurator/platforms.py new file mode 100644 index 0000000..d4ffe97 --- /dev/null +++ b/netbox_config_diff/configurator/platforms.py @@ -0,0 +1,122 @@ +import re +from typing import Pattern + +from scrapli_cfg.exceptions import TemplateError +from scrapli_cfg.platform.core.arista_eos import AsyncScrapliCfgEOS +from scrapli_cfg.platform.core.cisco_iosxe import AsyncScrapliCfgIOSXE +from scrapli_cfg.platform.core.cisco_iosxr import AsyncScrapliCfgIOSXR +from scrapli_cfg.platform.core.cisco_nxos import AsyncScrapliCfgNXOS +from scrapli_cfg.platform.core.juniper_junos import AsyncScrapliCfgJunos +from scrapli_cfg.response import ScrapliCfgResponse + + +class CustomScrapliCfg: + def _render_substituted_config( + self, config_template: str, substitutes: list[tuple[str, Pattern[str]]], source_config: str + ) -> str: + """ + Render a substituted configuration file + + Renders a configuration based on a user template, substitutes, and a target config from the + device. + + Args: + config_template: config file to use as the base for substitutions -- should contain + jinja2-like variables that will be replaced with data fetched from the source config + by the substitutes patterns + substitutes: tuple of name, pattern -- where name matches the jinja2-like variable in + the config_template file, and pattern is a compiled regular expression pattern to be + used to fetch that section from the source config + source_config: current source config to use in substitution process + + Returns: + None + + Raises: + TemplateError: if no substitute sections are provided + TemplateError: if a substitute pattern is not found in the config template + + """ + self.logger.debug("rendering substituted config") + + if not substitutes: + msg = "no substitutes provided..." + self.logger.critical(msg) + raise TemplateError(msg) + + replace_sections = [(name, re.search(pattern=pattern, string=source_config)) for name, pattern in substitutes] + + rendered_config = "" + for name, replace_section in replace_sections: + if not replace_section: + msg = f"substitution pattern {name} was unable to find a match in the target config" " source" + self.logger.critical(msg) + raise TemplateError(msg) + + replace_group = replace_section.group() + rendered_config = config_template.replace(f"{{{{ {name} }}}}", replace_group) + + # remove any totally empty lines (from bad regex, or just device spitting out lines w/ + # nothing on it + rendered_config = "\n".join(line for line in rendered_config.splitlines() if line) + + self.logger.debug("rendering substituted config complete") + + return rendered_config + + async def render_substituted_config( + self, + config_template: str, + substitutes: list[tuple[str, Pattern[str]]], + source: str = "running", + ) -> tuple[ScrapliCfgResponse, str]: + """ + Render a substituted configuration file + + Renders a configuration based on a user template, substitutes, and a target config from the + device. + + Args: + config_template: config file to use as the base for substitutions -- should contain + jinja2-like variables that will be replaced with data fetched from the source config + by the substitutes patterns + substitutes: tuple of name, pattern -- where name matches the jinja2-like variable in + the config_template file, and pattern is a compiled regular expression pattern to be + used to fetch that section from the source config + source: config source to use for the substitution efforts, typically running|startup + + Returns: + str: actual and substituted/rendered config + + Raises: + N/A + + """ + self.logger.info("fetching configuration and replacing with provided substitutes") + + source_config = await self.get_config(source=source) + return source_config, self._render_substituted_config( + config_template=config_template, + substitutes=substitutes, + source_config=source_config.result, + ) + + +class CustomAsyncScrapliCfgEOS(CustomScrapliCfg, AsyncScrapliCfgEOS): + pass + + +class CustomAsyncScrapliCfgIOSXE(CustomScrapliCfg, AsyncScrapliCfgIOSXE): + pass + + +class CustomAsyncScrapliCfgIOSXR(CustomScrapliCfg, AsyncScrapliCfgIOSXR): + pass + + +class CustomAsyncScrapliCfgNXOS(CustomScrapliCfg, AsyncScrapliCfgNXOS): + pass + + +class CustomAsyncScrapliCfgJunos(CustomScrapliCfg, AsyncScrapliCfgJunos): + pass diff --git a/netbox_config_diff/configurator/utils.py b/netbox_config_diff/configurator/utils.py new file mode 100644 index 0000000..c556f32 --- /dev/null +++ b/netbox_config_diff/configurator/utils.py @@ -0,0 +1,53 @@ +import logging + +from django.utils import timezone +from extras.choices import LogLevelChoices + + +class CustomLogger: + def __init__(self) -> None: + self.log_data = [] + self.diffs = [] + self.logger = logging.getLogger("netbox_config_diff.configurator") + + def _log(self, message: str, log_level: str | None = None) -> None: + if log_level not in LogLevelChoices.values(): + raise Exception(f"Unknown logging level: {log_level}") + if log_level is None: + log_level = LogLevelChoices.LOG_DEFAULT + self.log_data.append((timezone.now().strftime('%Y-%m-%d %H:%M:%S'), log_level, message)) + + def log(self, message: str) -> None: + self._log(message, log_level=LogLevelChoices.LOG_DEFAULT) + self.logger.info(message) + + def log_success(self, message: str) -> None: + self._log(message, log_level=LogLevelChoices.LOG_SUCCESS) + self.logger.info(message) + + def log_info(self, message: str) -> None: + self._log(message, log_level=LogLevelChoices.LOG_INFO) + self.logger.info(message) + + def log_warning(self, message: str) -> None: + self._log(message, log_level=LogLevelChoices.LOG_WARNING) + self.logger.info(message) + + def log_failure(self, message: str) -> None: + self._log(message, log_level=LogLevelChoices.LOG_FAILURE) + self.logger.info(message) + + def clear_log(self) -> None: + self.log_data = [] + + def logs(self) -> dict: + return {"logs": self.log_data} + + def get_diffs(self) -> dict: + return {"diffs": self.diffs} + + def get_data(self) -> dict: + return self.get_diffs() | self.logs() + + def add_diff(self, name: str, diff: str | None = None, error: str | None = None) -> None: + self.diffs.append({"name": name, "diff": diff, "error": error}) diff --git a/netbox_config_diff/constants.py b/netbox_config_diff/constants.py new file mode 100644 index 0000000..14f514b --- /dev/null +++ b/netbox_config_diff/constants.py @@ -0,0 +1,7 @@ +ACCEPTABLE_DRIVERS = [ + "arista_eos", + "cisco_iosxe", + "cisco_iosxr", + "cisco_nxos", + "juniper_junos", +] diff --git a/netbox_config_diff/filtersets.py b/netbox_config_diff/filtersets.py index 744e3a7..1bd32f0 100644 --- a/netbox_config_diff/filtersets.py +++ b/netbox_config_diff/filtersets.py @@ -1,10 +1,12 @@ import django_filters from dcim.models import Device, Platform +from django.contrib.auth import get_user_model from django.db.models import Q from netbox.filtersets import ChangeLoggedModelFilterSet, NetBoxModelFilterSet +from utilities.filters import MultiValueDateTimeFilter -from .choices import ConfigComplianceStatusChoices -from .models import ConfigCompliance, PlatformSetting +from netbox_config_diff.choices import ConfigComplianceStatusChoices +from netbox_config_diff.models import ConfigCompliance, ConfigurationRequest, PlatformSetting, Substitute class ConfigComplianceFilterSet(ChangeLoggedModelFilterSet): @@ -53,3 +55,63 @@ def search(self, queryset, name, value): | Q(description__icontains=value) ) return queryset.filter(qs_filter) + + +class ConfigurationRequestFilterSet(NetBoxModelFilterSet): + created_by_id = django_filters.ModelMultipleChoiceFilter( + queryset=get_user_model().objects.all(), + ) + created_by = django_filters.ModelMultipleChoiceFilter( + field_name="created_by__username", + queryset=get_user_model().objects.all(), + to_field_name="username", + ) + approved_by_id = django_filters.ModelMultipleChoiceFilter( + queryset=get_user_model().objects.all(), + ) + approved_by = django_filters.ModelMultipleChoiceFilter( + field_name="approved_by__username", + queryset=get_user_model().objects.all(), + to_field_name="username", + ) + scheduled_by_id = django_filters.ModelMultipleChoiceFilter( + queryset=get_user_model().objects.all(), + ) + scheduled_by = django_filters.ModelMultipleChoiceFilter( + field_name="scheduled_by__username", + queryset=get_user_model().objects.all(), + to_field_name="username", + ) + device_id = django_filters.ModelMultipleChoiceFilter( + field_name="devices", + queryset=Device.objects.all(), + ) + scheduled = MultiValueDateTimeFilter() + started = MultiValueDateTimeFilter() + completed = MultiValueDateTimeFilter() + + class Meta: + model = ConfigurationRequest + fields = ["id", "status", "description", "comments"] + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + qs_filter = Q(description__icontains=value) | Q(comments__icontains=value) + return queryset.filter(qs_filter) + + +class SubstituteFilterSet(NetBoxModelFilterSet): + platform_setting_id = django_filters.ModelMultipleChoiceFilter( + queryset=PlatformSetting.objects.all(), + ) + + class Meta: + model = Substitute + fields = ["id", "name", "description", "regexp"] + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + qs_filter = Q(name__icontains=value) | Q(description__icontains=value) | Q(regexp__icontains=value) + return queryset.filter(qs_filter) diff --git a/netbox_config_diff/forms.py b/netbox_config_diff/forms.py index 6d43f70..2682bf1 100644 --- a/netbox_config_diff/forms.py +++ b/netbox_config_diff/forms.py @@ -1,14 +1,20 @@ +from dcim.choices import DeviceStatusChoices from dcim.models import Device, Platform from django import forms +from django.contrib.auth import get_user_model from netbox.forms import NetBoxModelBulkEditForm, NetBoxModelFilterSetForm, NetBoxModelForm from utilities.forms.fields import ( DynamicModelChoiceField, DynamicModelMultipleChoiceField, TagFilterField, ) +from utilities.forms.mixins import BootstrapMixin +from utilities.forms.widgets import DateTimePicker +from utilities.utils import local_now -from .choices import ConfigComplianceStatusChoices -from .models import ConfigCompliance, PlatformSetting +from netbox_config_diff.choices import ConfigComplianceStatusChoices, ConfigurationRequestStatusChoices +from netbox_config_diff.constants import ACCEPTABLE_DRIVERS +from netbox_config_diff.models import ConfigCompliance, ConfigurationRequest, PlatformSetting, Substitute class ConfigComplianceFilterForm(NetBoxModelFilterSetForm): @@ -48,6 +54,7 @@ class PlatformSettingFilterForm(NetBoxModelFilterSetForm): platform_id = DynamicModelMultipleChoiceField( queryset=Platform.objects.all(), required=False, + label="Platform", ) tag = TagFilterField(model) @@ -73,3 +80,119 @@ class PlatformSettingBulkEditForm(NetBoxModelBulkEditForm): model = PlatformSetting fieldsets = ((None, ("driver", "command", "description", "exclude_regex")),) nullable_fields = ("description", "exclude_regex") + + +class ConfigurationRequestForm(NetBoxModelForm): + devices = DynamicModelMultipleChoiceField( + queryset=Device.objects.all(), + query_params={ + "status": DeviceStatusChoices.STATUS_ACTIVE, + "has_primary_ip": True, + "platform_id__n": "null", + "config_template_id__n": "null", + }, + ) + created_by = forms.ModelChoiceField( + queryset=get_user_model().objects.all(), + required=False, + widget=forms.HiddenInput(), + ) + + class Meta: + model = ConfigurationRequest + fields = ("devices", "description", "comments", "created_by", "tags") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if self.instance.pk: + self.fields["devices"].disabled = True + + def clean(self): + super().clean() + + if devices := self.cleaned_data["devices"].filter(platform__platform_setting__isnull=True): + platforms = {d.platform.name for d in devices} + raise forms.ValidationError({"devices": f"Assign PlatformSetting for platform(s): {', '.join(platforms)}"}) + + if drivers := { + device.platform.platform_setting.driver + for device in self.cleaned_data["devices"] + if device.platform.platform_setting.driver not in ACCEPTABLE_DRIVERS + }: + raise forms.ValidationError({"devices": f"Driver(s) not supported: {', '.join(drivers)}"}) + + +class ConfigurationRequestFilterForm(NetBoxModelFilterSetForm): + model = ConfigurationRequest + fieldsets = ((None, ("q", "created_by_id", "approved_by_id", "scheduled_by_id", "device_id", "status", "tag")),) + created_by_id = DynamicModelMultipleChoiceField( + queryset=get_user_model().objects.all(), + required=False, + label="Created by", + ) + approved_by_id = DynamicModelMultipleChoiceField( + queryset=get_user_model().objects.all(), + required=False, + label="Approved by", + ) + scheduled_by_id = DynamicModelMultipleChoiceField( + queryset=get_user_model().objects.all(), + required=False, + label="Scheduled by", + ) + device_id = DynamicModelMultipleChoiceField( + queryset=Device.objects.all(), + required=False, + label="Device", + ) + status = forms.MultipleChoiceField( + choices=ConfigurationRequestStatusChoices, + required=False, + ) + tag = TagFilterField(model) + + +class ConfigurationRequestScheduleForm(BootstrapMixin, forms.ModelForm): + scheduled = forms.DateTimeField( + widget=DateTimePicker(), + label="Schedule at", + help_text="Schedule execution of configuration request to a set time", + ) + scheduled_by = forms.ModelChoiceField( + queryset=get_user_model().objects.all(), + required=False, + widget=forms.HiddenInput(), + ) + + class Meta: + model = ConfigurationRequest + fields = ("scheduled", "scheduled_by", "status") + widgets = { + "status": forms.HiddenInput(), + } + + def clean(self): + scheduled_time = self.cleaned_data.get("scheduled") + if scheduled_time and scheduled_time < local_now(): + raise forms.ValidationError("Scheduled time must be in the future.") + + +class SubstituteForm(NetBoxModelForm): + platform_setting = DynamicModelChoiceField( + queryset=PlatformSetting.objects.all(), + ) + + class Meta: + model = Substitute + fields = ("platform_setting", "name", "description", "regexp", "tags") + + +class SubstituteFilterForm(NetBoxModelFilterSetForm): + model = Substitute + fieldsets = ((None, ("q", "platform_setting_id", "tag")),) + platform_setting_id = DynamicModelMultipleChoiceField( + queryset=PlatformSetting.objects.all(), + required=False, + ) + tag = TagFilterField(model) diff --git a/netbox_config_diff/graphql.py b/netbox_config_diff/graphql.py index 600df91..33fde3d 100644 --- a/netbox_config_diff/graphql.py +++ b/netbox_config_diff/graphql.py @@ -2,7 +2,7 @@ from netbox.graphql.fields import ObjectField, ObjectListField from netbox.graphql.types import NetBoxObjectType -from . import filtersets, models +from netbox_config_diff import filtersets, models class ConfigComplianceType(NetBoxObjectType): @@ -19,6 +19,20 @@ class Meta: filterset_class = filtersets.PlatformSettingFilterSet +class ConfigurationRequestType(NetBoxObjectType): + class Meta: + model = models.ConfigurationRequest + fields = "__all__" + filterset_class = filtersets.ConfigurationRequestFilterSet + + +class SubstituteType(NetBoxObjectType): + class Meta: + model = models.Substitute + fields = "__all__" + filterset_class = filtersets.SubstituteFilterSet + + class Query(ObjectType): config_compliance = ObjectField(ConfigComplianceType) config_compliance_list = ObjectListField(ConfigComplianceType) @@ -26,5 +40,11 @@ class Query(ObjectType): platform_setting = ObjectField(PlatformSettingType) platform_setting_list = ObjectListField(PlatformSettingType) + configuration_request = ObjectField(ConfigurationRequestType) + configuration_request_list = ObjectListField(ConfigurationRequestType) + + substitute = ObjectField(SubstituteType) + substitute_list = ObjectListField(SubstituteType) + schema = Query diff --git a/netbox_config_diff/jobs.py b/netbox_config_diff/jobs.py new file mode 100644 index 0000000..873414d --- /dev/null +++ b/netbox_config_diff/jobs.py @@ -0,0 +1,52 @@ +import logging +import traceback + +from core.choices import JobStatusChoices +from core.models import Job +from utilities.utils import NetBoxFakeRequest + +from netbox_config_diff.choices import ConfigurationRequestStatusChoices +from netbox_config_diff.configurator.base import Configurator +from netbox_config_diff.models import ConfigurationRequest + +logger = logging.getLogger(__name__) + + +def collect_diffs(job: Job, request: NetBoxFakeRequest, *args, **kwargs) -> None: + job.start() + cr = ConfigurationRequest.objects.get(pk=job.object_id) + logger.info(f"Collecting diffs for {cr}") + configurator = Configurator(cr.devices.all(), request) + try: + configurator.validate_devices() + configurator.collect_diffs() + job.data = configurator.logger.get_data() + job.terminate() + except Exception as e: + stacktrace = traceback.format_exc() + configurator.logger.log_failure(f"An exception occurred: `{type(e).__name__}: {e}`\n```\n{stacktrace}\n```") + logger.error(f"Exception raised during script execution: {e}") + job.data = configurator.logger.get_data() + job.terminate(status=JobStatusChoices.STATUS_ERRORED) + + logger.info(f"Collecting diffs job completed in {job.duration}") + + +def push_configs(job: Job, request: NetBoxFakeRequest, *args, **kwargs) -> None: + cr = ConfigurationRequest.objects.get(pk=job.object_id) + cr.start(job) + logger.info(f"Applying configs for {cr}") + configurator = Configurator(cr.devices.all(), request) + try: + configurator.validate_devices() + configurator.push_configs() + job.data = configurator.logger.get_data() + cr.terminate(job=job) + except Exception as e: + stacktrace = traceback.format_exc() + configurator.logger.log_failure(f"An exception occurred: `{type(e).__name__}: {e}`\n```\n{stacktrace}\n```") + logger.error(f"Exception raised during script execution: {e}") + job.data = configurator.logger.get_data() + cr.terminate(job=job, status=ConfigurationRequestStatusChoices.ERRORED) + + logger.info(f"Applying configs job completed in {job.duration}") diff --git a/netbox_config_diff/migrations/0006_substitute.py b/netbox_config_diff/migrations/0006_substitute.py new file mode 100644 index 0000000..4825db6 --- /dev/null +++ b/netbox_config_diff/migrations/0006_substitute.py @@ -0,0 +1,62 @@ +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import re +import taggit.managers +import utilities.json + + +class Migration(migrations.Migration): + dependencies = [ + ('extras', '0098_webhook_custom_field_data_webhook_tags'), + ('netbox_config_diff', '0005_configcompliance_extra_missing'), + ] + + operations = [ + migrations.CreateModel( + name='Substitute', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ( + 'custom_field_data', + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ( + 'name', + models.CharField( + max_length=250, + unique=True, + validators=[ + django.core.validators.RegexValidator( + flags=re.RegexFlag['IGNORECASE'], + message='Only alphanumeric characters and underscores are allowed.', + regex='^[a-z0-9_]+$', + ), + django.core.validators.RegexValidator( + flags=re.RegexFlag['IGNORECASE'], + inverse_match=True, + message='Double underscores are not permitted in names.', + regex='__', + ), + ], + ), + ), + ('description', models.CharField(blank=True, max_length=200)), + ('regexp', models.CharField(max_length=1000)), + ( + 'platform_setting', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='substitutes', + to='netbox_config_diff.platformsetting', + ), + ), + ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ], + options={ + 'ordering': ('name',), + }, + ), + ] diff --git a/netbox_config_diff/migrations/0007_configurationrequest.py b/netbox_config_diff/migrations/0007_configurationrequest.py new file mode 100644 index 0000000..1241ca0 --- /dev/null +++ b/netbox_config_diff/migrations/0007_configurationrequest.py @@ -0,0 +1,70 @@ +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import taggit.managers +import utilities.json + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('extras', '0098_webhook_custom_field_data_webhook_tags'), + ('dcim', '0181_rename_device_role_device_role'), + ('netbox_config_diff', '0006_substitute'), + ] + + operations = [ + migrations.CreateModel( + name='ConfigurationRequest', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ( + 'custom_field_data', + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ('description', models.CharField(blank=True, max_length=200)), + ('comments', models.TextField(blank=True)), + ('status', models.CharField(default='created', max_length=30)), + ('scheduled', models.DateTimeField(blank=True, null=True)), + ('started', models.DateTimeField(blank=True, null=True)), + ('completed', models.DateTimeField(blank=True, null=True)), + ( + 'approved_by', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='+', + to=settings.AUTH_USER_MODEL, + ), + ), + ( + 'created_by', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='+', + to=settings.AUTH_USER_MODEL, + ), + ), + ('devices', models.ManyToManyField(related_name='configuration_requests', to='dcim.device')), + ( + 'scheduled_by', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='+', + to=settings.AUTH_USER_MODEL, + ), + ), + ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ], + options={ + 'ordering': ('-created',), + }, + ), + ] diff --git a/netbox_config_diff/models.py b/netbox_config_diff/models.py index 82d41c5..d65dff1 100644 --- a/netbox_config_diff/models.py +++ b/netbox_config_diff/models.py @@ -1,11 +1,22 @@ +import re + +import django_rq +from core.models import Job +from django.conf import settings +from django.core.validators import RegexValidator from django.db import models from django.urls import reverse +from django.utils import timezone +from django.utils.module_loading import import_string from django.utils.translation import gettext as _ -from netbox.models import NetBoxModel -from netbox.models.features import ChangeLoggingMixin +from netbox.constants import RQ_QUEUE_DEFAULT +from netbox.models import NetBoxModel, PrimaryModel +from netbox.models.features import ChangeLoggingMixin, JobsMixin +from rq.exceptions import InvalidJobOperation from utilities.querysets import RestrictedQuerySet +from utilities.utils import copy_safe_request -from .choices import ConfigComplianceStatusChoices +from netbox_config_diff.choices import ConfigComplianceStatusChoices, ConfigurationRequestStatusChoices class ConfigCompliance(ChangeLoggingMixin, models.Model): @@ -96,3 +107,144 @@ def __str__(self): def get_absolute_url(self): return reverse("plugins:netbox_config_diff:platformsetting", args=[self.pk]) + + +class ConfigurationRequest(JobsMixin, PrimaryModel): + created_by = models.ForeignKey( + to=settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + related_name='+', + blank=True, + null=True, + ) + approved_by = models.ForeignKey( + to=settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + related_name='+', + blank=True, + null=True, + ) + scheduled_by = models.ForeignKey( + to=settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + related_name='+', + blank=True, + null=True, + ) + status = models.CharField( + max_length=30, + choices=ConfigurationRequestStatusChoices, + default=ConfigurationRequestStatusChoices.CREATED, + ) + devices = models.ManyToManyField( + to="dcim.Device", + related_name="configuration_requests", + ) + scheduled = models.DateTimeField( + null=True, + blank=True, + ) + started = models.DateTimeField( + null=True, + blank=True, + ) + completed = models.DateTimeField( + null=True, + blank=True, + ) + + class Meta: + ordering = ("-created",) + + def __str__(self): + return f"CR #{self.pk}" + + def get_absolute_url(self): + return reverse("plugins:netbox_config_diff:configurationrequest", args=[self.pk]) + + def get_status_color(self): + return ConfigurationRequestStatusChoices.colors.get(self.status) + + @property + def finished(self): + return self.status in ConfigurationRequestStatusChoices.FINISHED_STATE_CHOICES + + def delete(self, *args, **kwargs): + super().delete(*args, **kwargs) + + queue = django_rq.get_queue(RQ_QUEUE_DEFAULT) + for result in self.jobs.all(): + if job := queue.fetch_job(str(result.job_id)): + try: + job.cancel() + except InvalidJobOperation: + pass + + def enqueue_job(self, request, job_name, schedule_at=None): + return Job.enqueue( + import_string(f"netbox_config_diff.jobs.{job_name}"), + name=f"{self} {job_name}", + instance=self, + user=request.user, + request=copy_safe_request(request), + schedule_at=schedule_at, + ) + + def start(self, job: Job): + """ + Record the job's start time and update its status to "running." + """ + if self.started is not None: + return + job.start() + self.started = timezone.now() + self.status = ConfigurationRequestStatusChoices.RUNNING + self.save() + + def terminate(self, job: Job, status: str = ConfigurationRequestStatusChoices.COMPLETED): + job.terminate(status=status) + self.status = status + self.completed = timezone.now() + self.save() + + +class Substitute(NetBoxModel): + platform_setting = models.ForeignKey( + to="netbox_config_diff.PlatformSetting", + on_delete=models.CASCADE, + related_name="substitutes", + ) + name = models.CharField( + max_length=250, + unique=True, + validators=( + RegexValidator( + regex=r'^[a-z0-9_]+$', + message=_("Only alphanumeric characters and underscores are allowed."), + flags=re.IGNORECASE, + ), + RegexValidator( + regex=r'__', + message=_("Double underscores are not permitted in names."), + flags=re.IGNORECASE, + inverse_match=True, + ), + ), + ) + description = models.CharField( + verbose_name=_('description'), + max_length=200, + blank=True, + ) + regexp = models.CharField( + max_length=1000, + ) + + class Meta: + ordering = ("name",) + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse("plugins:netbox_config_diff:substitute", args=[self.pk]) diff --git a/netbox_config_diff/navigation.py b/netbox_config_diff/navigation.py index 5f5a6ae..0e08eb1 100644 --- a/netbox_config_diff/navigation.py +++ b/netbox_config_diff/navigation.py @@ -1,3 +1,6 @@ +from django import forms +from extras.dashboard.utils import register_widget +from extras.dashboard.widgets import DashboardWidget, WidgetConfigForm from extras.plugins import PluginMenuButton, PluginMenuItem from utilities.choices import ButtonColorChoices @@ -25,4 +28,34 @@ def get_add_button(model: str) -> PluginMenuButton: buttons=[], permissions=["netbox_config_diff.view_configcompliance"], ), + PluginMenuItem( + link="plugins:netbox_config_diff:configurationrequest_list", + link_text="Configuration Requests", + buttons=[get_add_button("configurationrequest")], + permissions=["netbox_config_diff.view_configurationrequest"], + ), + PluginMenuItem( + link="plugins:netbox_config_diff:configurationrequest_job_list", + link_text="Jobs", + buttons=[], + permissions=["core.view_job"], + ), + PluginMenuItem( + link="plugins:netbox_config_diff:substitute_list", + link_text="Substitutes", + buttons=[get_add_button("substitute")], + permissions=["netbox_config_diff.view_substitute"], + ), ) + + +@register_widget +class ReminderWidget(DashboardWidget): + default_title = 'Reminder' + description = 'Add a virtual sticky note' + + class ConfigForm(WidgetConfigForm): + content = forms.CharField(widget=forms.Textarea()) + + def render(self, request): + return self.config.get('content') diff --git a/netbox_config_diff/search.py b/netbox_config_diff/search.py index 9e4428b..e5e8d67 100644 --- a/netbox_config_diff/search.py +++ b/netbox_config_diff/search.py @@ -1,6 +1,6 @@ from netbox.search import SearchIndex, register_search -from . import models +from netbox_config_diff import models @register_search @@ -11,3 +11,22 @@ class PlatformSettingIndex(SearchIndex): ("command", 500), ("exclude_regex", 1000), ) + + +@register_search +class ConfigurationRequestIndex(SearchIndex): + model = models.ConfigurationRequest + fields = ( + ("description", 100), + ("comments", 500), + ) + + +@register_search +class SubstituteIndex(SearchIndex): + model = models.Substitute + fields = ( + ("name", 100), + ("description", 500), + ("regexp", 1000), + ) diff --git a/netbox_config_diff/tables.py b/netbox_config_diff/tables.py index 1afee66..ea3fa01 100644 --- a/netbox_config_diff/tables.py +++ b/netbox_config_diff/tables.py @@ -1,7 +1,7 @@ import django_tables2 as tables from netbox.tables import NetBoxTable, columns -from .models import ConfigCompliance, PlatformSetting +from netbox_config_diff.models import ConfigCompliance, ConfigurationRequest, PlatformSetting, Substitute class ConfigComplianceTable(NetBoxTable): @@ -34,3 +34,81 @@ class Meta(NetBoxTable.Meta): model = PlatformSetting fields = ("driver", "platform", "command", "exclude_regex", "description", "tags", "created", "last_updated") default_columns = ("driver", "platform", "command", "exclude_regex", "description") + + +class ConfigurationRequestTable(NetBoxTable): + devices = columns.ManyToManyColumn( + linkify_item=True, + ) + status = columns.ChoiceFieldColumn() + scheduled = columns.DateTimeColumn() + started = columns.DateTimeColumn() + completed = columns.DateTimeColumn() + tags = columns.TagColumn( + url_name="netbox_config_diff:configurationrequest_list", + ) + actions = columns.ActionsColumn( + actions=("delete", "changelog"), + ) + + class Meta(NetBoxTable.Meta): + model = ConfigurationRequest + fields = ( + "id", + "devices", + "status", + "description", + "created_by", + "approved_by", + "scheduled_by", + "scheduled", + "started", + "completed", + "tags", + "created", + "last_updated", + ) + default_columns = ( + "id", + "devices", + "status", + "description", + "created_by", + "approved_by", + "scheduled_by", + "scheduled", + "started", + "completed", + ) + + +class SubstituteTable(NetBoxTable): + name = tables.Column( + linkify=True, + ) + platform_setting = tables.Column( + linkify=True, + ) + tags = columns.TagColumn( + url_name="netbox_config_diff:substitute_list", + ) + + class Meta(NetBoxTable.Meta): + model = Substitute + fields = ( + "id", + "name", + "platform_setting", + "description", + "regexp", + "tags", + "created", + "last_updated", + ) + default_columns = ( + "name", + "platform_setting", + "description", + "regexp", + "tags", + ) diff --git a/netbox_config_diff/templates/netbox_config_diff/configurationrequest.html b/netbox_config_diff/templates/netbox_config_diff/configurationrequest.html new file mode 100644 index 0000000..8798f84 --- /dev/null +++ b/netbox_config_diff/templates/netbox_config_diff/configurationrequest.html @@ -0,0 +1,117 @@ +{% extends "netbox_config_diff/configurationrequest/base.html" %} +{% load helpers %} + +{% block content %} +
+
+
+
{{ object|meta:"verbose_name"|bettertitle }}
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Status{% badge object.get_status_display bg_color=object.get_status_color %}
Description{{ object.description|placeholder }}
Created by + {% if object.created_by %} + {% if object.created_by.get_full_name %} + {{ object.created_by.get_full_name }} + {% else %} + {{ object.created_by.username }} + {% endif %} + {% else %} + — + {% endif %} +
Approved by + {% if object.approved_by %} + {% if object.approved_by.get_full_name %} + {{ object.approved_by.get_full_name }} + {% else %} + {{ object.approved_by.username }} + {% endif %} + {% else %} + — + {% endif %} +
Scheduled by + {% if object.scheduled_by %} + {% if object.scheduled_by.get_full_name %} + {{ object.scheduled_by.get_full_name }} + {% else %} + {{ object.scheduled_by.username }} + {% endif %} + {% else %} + — + {% endif %} +
Scheduled{{ object.scheduled|annotated_date|placeholder }}
Started{{ object.started|annotated_date|placeholder }}
Completed{{ object.completed|annotated_date|placeholder }}
+
+
+ {% include 'inc/panels/comments.html' %} + {% include "inc/panels/tags.html" %} + {% include "inc/panels/custom_fields.html" %} +
+
+
+
+ Devices +
+
+ + + + + + + + + {% for device in object.devices.all %} + + + + + {% empty %} + + {% endfor %} + +
NameIP Address
{{ device|linkify }}{{ device.primary_ip|linkify }}
+ None +
+
+
+
+
+ {% if job %} +
+
+ {% include 'netbox_config_diff/inc/job_log.html' %} +
+
+ {% endif %} +{% endblock content %} diff --git a/netbox_config_diff/templates/netbox_config_diff/configurationrequest/base.html b/netbox_config_diff/templates/netbox_config_diff/configurationrequest/base.html new file mode 100644 index 0000000..11e2303 --- /dev/null +++ b/netbox_config_diff/templates/netbox_config_diff/configurationrequest/base.html @@ -0,0 +1,57 @@ +{% extends "generic/object.html" %} +{% load buttons %} +{% load perms %} + +{% block controls %} +
+
+ {% if perms.extras.add_bookmark and object.bookmarks %} + {% bookmark_button object %} + {% endif %} + {% if not object.finished %} +
+ {% csrf_token %} + +
+ {% if perms.netbox_config_diff.approve_configurationrequest %} + {% if not object.approved_by %} +
+ {% csrf_token %} + +
+ {% endif %} + {% if object.approved_by %} +
+ {% csrf_token %} + +
+ {% if object.scheduled_by %} +
+ {% csrf_token %} + +
+ {% else %} + + Schedule + + {% endif %} + {% endif %} + {% endif %} + {% if request.user|can_change:object %} + {% edit_button object %} + {% endif %} + {% endif %} + {% if request.user|can_delete:object %} + {% delete_button object %} + {% endif %} +
+
+{% endblock controls %} diff --git a/netbox_config_diff/templates/netbox_config_diff/configurationrequest/diffs.html b/netbox_config_diff/templates/netbox_config_diff/configurationrequest/diffs.html new file mode 100644 index 0000000..0c89ab1 --- /dev/null +++ b/netbox_config_diff/templates/netbox_config_diff/configurationrequest/diffs.html @@ -0,0 +1,102 @@ +{% extends "netbox_config_diff/configurationrequest/base.html" %} +{% load helpers %} +{% load static %} + +{% block content %} + {% if job %} +
+
+
+
Job
+
+ + + + + + + + + + + + + +
Name + {{ job.name }} +
Status{% badge job.get_status_display job.get_status_color %}
Created By{{ job.user|placeholder }}
+
+
+
+
+
+
Time
+
+ + + + + + + + + + + + + +
Created{{ job.created|annotated_date }}
Started{{ job.started|annotated_date|placeholder }}
Completed{{ job.completed|annotated_date|placeholder }}
+
+
+
+
+ {% if job.completed %} +
+
+ {% if job.status == "failed" or job.status == "errored" %} + {% include 'netbox_config_diff/inc/job_log.html' %} + {% else %} + {% for diff in job.data.diffs %} + {% if diff.error %} +
+
{{ diff.name }} - Error
+
+ 
{{ diff.error }}
+
+
+ {% elif diff.diff %} + {% include 'netbox_config_diff/inc/diff.html' with device_name=diff.name data=diff.diff %} + {% else %} +
+
{{ diff.name }} - No diff
+
+ {% endif %} + {% endfor %} + {% endif %} +
+
+ {% endif %} + {% else %} + + {% endif %} +{% endblock content %} + +{% block javascript %} + {% if job.status == "completed" %} + + + {% endif %} +{% endblock javascript %} diff --git a/netbox_config_diff/templates/netbox_config_diff/inc/diff.html b/netbox_config_diff/templates/netbox_config_diff/inc/diff.html new file mode 100644 index 0000000..bc1864a --- /dev/null +++ b/netbox_config_diff/templates/netbox_config_diff/inc/diff.html @@ -0,0 +1,27 @@ +
+
{{ device_name }} - Diff
+
+
+ diff --git a/netbox_config_diff/templates/netbox_config_diff/inc/job_log.html b/netbox_config_diff/templates/netbox_config_diff/inc/job_log.html new file mode 100644 index 0000000..4e0c508 --- /dev/null +++ b/netbox_config_diff/templates/netbox_config_diff/inc/job_log.html @@ -0,0 +1,27 @@ +{% load log_levels %} + +
+
Job Log
+
+ + + + + + + {% for log in job.data.logs %} + + + + + + {% empty %} + + + + {% endfor %} +
TimeLevelMessage
{{ log.0 }}{% log_level log.1 %}{{ log.2|markdown }}
+ No log output +
+
+
diff --git a/netbox_config_diff/templates/netbox_config_diff/platformsetting.html b/netbox_config_diff/templates/netbox_config_diff/platformsetting.html index 306ee6f..4bdf7f5 100644 --- a/netbox_config_diff/templates/netbox_config_diff/platformsetting.html +++ b/netbox_config_diff/templates/netbox_config_diff/platformsetting.html @@ -1,4 +1,3 @@ - {% extends "generic/object.html" %} {% load helpers %} {% load render_table from django_tables2 %} @@ -46,4 +45,3 @@
{{ object|meta:"verbose_name"|bettertitle }}
{% endblock content %} - diff --git a/netbox_config_diff/templates/netbox_config_diff/substitute.html b/netbox_config_diff/templates/netbox_config_diff/substitute.html new file mode 100644 index 0000000..51b940c --- /dev/null +++ b/netbox_config_diff/templates/netbox_config_diff/substitute.html @@ -0,0 +1,37 @@ +{% extends "generic/object.html" %} +{% load helpers %} +{% load render_table from django_tables2 %} + +{% block content %} +
+
+
+
{{ object|meta:"verbose_name"|bettertitle }}
+
+ + + + + + + + + + + + + + + + + +
Platform Setting{{ object.platform_setting|linkify }}
Name{{ object.name }}
Description{{ object.description|placeholder }}
Regexp
{{ object.regexp }}
+
+
+ {% include "inc/panels/custom_fields.html" %} +
+
+ {% include "inc/panels/tags.html" %} +
+
+{% endblock content %} diff --git a/netbox_config_diff/urls.py b/netbox_config_diff/urls.py index 14dafba..b9d22c8 100644 --- a/netbox_config_diff/urls.py +++ b/netbox_config_diff/urls.py @@ -1,7 +1,7 @@ from django.urls import include, path from utilities.urls import get_model_urls -from . import views +from netbox_config_diff import views urlpatterns = ( # Config Compliances @@ -20,4 +20,14 @@ "platform-settings/delete/", views.PlatformSettingBulkDeleteView.as_view(), name="platformsetting_bulk_delete" ), path("platform-settings//", include(get_model_urls("netbox_config_diff", "platformsetting"))), + # Configuration Requests + path("configuration-requests/", views.ConfigurationRequestListView.as_view(), name="configurationrequest_list"), + path("configuration-requests/add/", views.ConfigurationRequestEditView.as_view(), name="configurationrequest_add"), + path("configuration-requests//", include(get_model_urls("netbox_config_diff", "configurationrequest"))), + # Jobs + path("jobs/", views.JobListView.as_view(), name="configurationrequest_job_list"), + # Configuration Requests + path("substitutes/", views.SubstituteListView.as_view(), name="substitute_list"), + path("substitutes/add/", views.SubstituteEditView.as_view(), name="substitute_add"), + path("substitutes//", include(get_model_urls("netbox_config_diff", "substitute"))), ) diff --git a/netbox_config_diff/views/__init__.py b/netbox_config_diff/views/__init__.py new file mode 100644 index 0000000..fc696be --- /dev/null +++ b/netbox_config_diff/views/__init__.py @@ -0,0 +1,29 @@ +from .compliance import ( + ConfigComplianceBulkDeleteView, + ConfigComplianceListView, + PlatformSettingBulkDeleteView, + PlatformSettingBulkEditView, + PlatformSettingEditView, + PlatformSettingListView, +) +from .configuration import ( + ConfigurationRequestEditView, + ConfigurationRequestListView, + JobListView, + SubstituteEditView, + SubstituteListView, +) + +__all__ = ( + "ConfigComplianceBulkDeleteView", + "ConfigComplianceListView", + "ConfigurationRequestEditView", + "ConfigurationRequestListView", + "JobListView", + "PlatformSettingBulkDeleteView", + "PlatformSettingBulkEditView", + "PlatformSettingEditView", + "PlatformSettingListView", + "SubstituteEditView", + "SubstituteListView", +) diff --git a/netbox_config_diff/views.py b/netbox_config_diff/views/compliance.py similarity index 94% rename from netbox_config_diff/views.py rename to netbox_config_diff/views/compliance.py index 4d486b3..f6cfd40 100644 --- a/netbox_config_diff/views.py +++ b/netbox_config_diff/views/compliance.py @@ -4,15 +4,15 @@ from netbox.views import generic from utilities.views import ViewTab, register_model_view -from .filtersets import ConfigComplianceFilterSet, PlatformSettingFilterSet -from .forms import ( +from netbox_config_diff.filtersets import ConfigComplianceFilterSet, PlatformSettingFilterSet +from netbox_config_diff.forms import ( ConfigComplianceFilterForm, PlatformSettingBulkEditForm, PlatformSettingFilterForm, PlatformSettingForm, ) -from .models import ConfigCompliance, PlatformSetting -from .tables import ConfigComplianceTable, PlatformSettingTable +from netbox_config_diff.models import ConfigCompliance, PlatformSetting +from netbox_config_diff.tables import ConfigComplianceTable, PlatformSettingTable class BaseConfigComplianceConfigView(generic.ObjectView): diff --git a/netbox_config_diff/views/configuration.py b/netbox_config_diff/views/configuration.py new file mode 100644 index 0000000..70d86d9 --- /dev/null +++ b/netbox_config_diff/views/configuration.py @@ -0,0 +1,332 @@ +import django_rq +from core.choices import JobStatusChoices +from core.filtersets import JobFilterSet +from core.forms import JobFilterForm +from core.models import Job +from core.tables import JobTable +from django.contrib import messages +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.http import HttpResponseForbidden +from django.shortcuts import get_object_or_404, redirect, render +from netbox.constants import RQ_QUEUE_DEFAULT +from netbox.views import generic +from netbox.views.generic.base import BaseObjectView +from rq.exceptions import InvalidJobOperation +from utilities.forms import restrict_form_fields +from utilities.rqworker import get_workers_for_queue +from utilities.utils import normalize_querydict +from utilities.views import ViewTab, register_model_view + +from netbox_config_diff.choices import ConfigurationRequestStatusChoices +from netbox_config_diff.filtersets import ConfigurationRequestFilterSet, SubstituteFilterSet +from netbox_config_diff.forms import ( + ConfigurationRequestFilterForm, + ConfigurationRequestForm, + ConfigurationRequestScheduleForm, + SubstituteFilterForm, + SubstituteForm, +) +from netbox_config_diff.models import ConfigurationRequest, Substitute +from netbox_config_diff.tables import ConfigurationRequestTable, SubstituteTable + + +@register_model_view(ConfigurationRequest) +class ConfigurationRequestView(generic.ObjectView): + queryset = ConfigurationRequest.objects.all() + + def get_extra_context(self, request, instance): + job = Job.objects.filter( + object_id=instance.pk, name__contains="push_configs", status__in=JobStatusChoices.TERMINAL_STATE_CHOICES + ).first() + + return { + "job": job, + } + + +class ConfigurationRequestListView(generic.ObjectListView): + queryset = ConfigurationRequest.objects.prefetch_related( + "devices", "created_by", "approved_by", "scheduled_by", "tags" + ) + filterset = ConfigurationRequestFilterSet + filterset_form = ConfigurationRequestFilterForm + table = ConfigurationRequestTable + + +@register_model_view(ConfigurationRequest, "edit") +class ConfigurationRequestEditView(generic.ObjectEditView): + queryset = ConfigurationRequest.objects.all() + form = ConfigurationRequestForm + + def get(self, request, *args, **kwargs): + obj = self.get_object(**kwargs) + if obj.finished: + messages.error(request, f"{obj} is finished, you can't change it.") + return redirect(obj.get_absolute_url()) + obj = self.alter_object(obj, request, args, kwargs) + model = self.queryset.model + + initial_data = normalize_querydict(request.GET) + initial_data["created_by"] = request.user.pk + form = self.form(instance=obj, initial=initial_data) + restrict_form_fields(form, request.user) + + return render( + request, + self.template_name, + { + "model": model, + "object": obj, + "form": form, + "return_url": self.get_return_url(request, obj), + **self.get_extra_context(request, obj), + }, + ) + + +@register_model_view(ConfigurationRequest, "delete") +class ConfigurationRequestDeleteView(generic.ObjectDeleteView): + queryset = ConfigurationRequest.objects.all() + + +@register_model_view(ConfigurationRequest, "diffs") +class ConfigurationRequestDiffsView(generic.ObjectView): + queryset = ConfigurationRequest.objects.all() + template_name = "netbox_config_diff/configurationrequest/diffs.html" + tab = ViewTab( + label="Diffs", + permission="netbox_config_diff.view_configurationrequest", + weight=500, + ) + + def get_extra_context(self, request, instance): + job = Job.objects.filter(object_id=instance.pk, name__contains="collect_diffs").first() + + return { + "job": job, + } + + +@register_model_view(ConfigurationRequest, "scheduled_job", "scheduled-job") +class ConfigurationRequestScheduledJobView(generic.ObjectChildrenView): + queryset = ConfigurationRequest.objects.all() + child_model = Job + table = JobTable + template_name = "generic/object_children.html" + tab = ViewTab( + label="Scheduled job", + badge=lambda obj: obj.jobs.filter( + object_id=obj.pk, name__contains="push_configs", status=JobStatusChoices.STATUS_SCHEDULED + ).count(), + permission="netbox_config_diff.view_configurationrequest", + weight=510, + hide_if_empty=True, + ) + + def get_children(self, request, parent): + return Job.objects.restrict(request.user, "view").filter( + object_id=parent.pk, name__contains="push_configs", status=JobStatusChoices.STATUS_SCHEDULED + ) + + def get_permitted_actions(self, user, model=None): + return [] + + def get_table(self, data, request, bulk_actions=True): + table = self.table(data, user=request.user, exclude=("actions",)) + table.configure(request) + + return table + + +@register_model_view(ConfigurationRequest, "approve") +class ConfigurationRequestApproveView(BaseObjectView): + queryset = ConfigurationRequest.objects.all() + + def get_required_permission(self): + return "netbox_config_diff.approve_configurationrequest" + + def get(self, request, pk): + obj = get_object_or_404(self.queryset, pk=pk) + return redirect(obj.get_absolute_url()) + + def post(self, request, pk): + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + messages.error(request, f"{obj} is finished, you can't change it.") + return redirect(obj.get_absolute_url()) + + if obj.approved_by: + obj.approved_by = None + obj.status = ConfigurationRequestStatusChoices.CREATED + if obj.scheduled: + obj.scheduled = None + obj.scheduled_by = None + messages.success(request, f"Unapproved {obj}") + else: + obj.approved_by = User.objects.filter(pk=request.user.pk).first() + obj.status = ConfigurationRequestStatusChoices.APPROVED + messages.success(request, f"Approved {obj}") + obj.save() + + return redirect(obj.get_absolute_url()) + + +@register_model_view(ConfigurationRequest, "schedule") +class ConfigurationRequestScheduleView(generic.ObjectEditView): + queryset = ConfigurationRequest.objects.all() + form = ConfigurationRequestScheduleForm + + def get_required_permission(self): + return "netbox_config_diff.approve_configurationrequest" + + def get(self, request, *args, **kwargs): + obj = self.get_object(**kwargs) + if obj.finished: + messages.error(request, f"{obj} is finished, you can't change it.") + return redirect(obj.get_absolute_url()) + if obj.scheduled: + messages.error(request, f"{obj} already scheduled.") + return redirect(obj.get_absolute_url()) + obj = self.alter_object(obj, request, args, kwargs) + model = self.queryset.model + + initial_data = normalize_querydict(request.GET) + initial_data["scheduled_by"] = request.user.pk + initial_data["status"] = ConfigurationRequestStatusChoices.SCHEDULED + form = self.form(instance=obj, initial=initial_data) + restrict_form_fields(form, request.user) + + return render( + request, + self.template_name, + { + "model": model, + "object": obj, + "form": form, + "return_url": self.get_return_url(request, obj), + **self.get_extra_context(request, obj), + }, + ) + + def post(self, request, pk): + if not request.user.has_perm("netbox_config_diff.approve_configurationrequest"): + return HttpResponseForbidden() + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + messages.error(request, f"{obj} is finished, you can't change it.") + elif not get_workers_for_queue("default"): + messages.error(request, "Unable to run script: RQ worker process not running.") + elif obj.scheduled_by: + messages.error(request, f"{obj} already scheduled.") + elif obj.approved_by is None: + messages.error(request, f"Approve {obj} before schedule.") + else: + form = self.form(data=request.POST, files=request.FILES, instance=obj) + if not form.is_valid(): + return render( + request, + self.template_name, + context={ + "object": obj, + "form": form, + "return_url": self.get_return_url(request, obj), + **self.get_extra_context(request, obj), + }, + ) + form.save() + obj.enqueue_job(request, "push_configs", schedule_at=form.cleaned_data["scheduled"]) + messages.success(request, f"Scheduled job for {obj}") + return redirect(obj.get_absolute_url()) + + +@register_model_view(ConfigurationRequest, "unschedule") +class ConfigurationRequestUnscheduleView(BaseObjectView): + queryset = ConfigurationRequest.objects.all() + + def get_required_permission(self): + return "netbox_config_diff.approve_configurationrequest" + + def get(self, request, pk): + obj = get_object_or_404(self.queryset, pk=pk) + return redirect(obj.get_absolute_url()) + + def post(self, request, pk): + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + messages.error(request, f"{obj} is finished, you can't change it.") + return redirect(obj.get_absolute_url()) + + if obj.scheduled_by: + obj.scheduled = None + obj.scheduled_by = None + obj.status = ConfigurationRequestStatusChoices.APPROVED + obj.save() + queue = django_rq.get_queue(RQ_QUEUE_DEFAULT) + for result in obj.jobs.filter(name__contains="push_configs", status=JobStatusChoices.STATUS_SCHEDULED): + result.delete() + if job := queue.fetch_job(str(result.job_id)): + try: + job.cancel() + except InvalidJobOperation: + pass + messages.success(request, f"Unscheduled {obj}") + + return redirect(obj.get_absolute_url()) + + +@register_model_view(ConfigurationRequest, name="collectdiffs", path="collect-diffs") +class ConfigurationRequestCollectDiffsView(BaseObjectView): + queryset = ConfigurationRequest.objects.all() + + def get_required_permission(self): + return "netbox_config_diff.change_configurationrequest" + + def get(self, request, pk): + obj = get_object_or_404(self.queryset, pk=pk) + return redirect(obj.get_absolute_url()) + + def post(self, request, pk): + obj = get_object_or_404(self.queryset, pk=pk) + if obj.finished: + messages.error(request, f"{obj} is finished, you can't change it.") + elif not get_workers_for_queue("default"): + messages.error(request, "Unable to run: RQ worker process not running.") + else: + obj.enqueue_job(request, "collect_diffs") + messages.success(request, f"Start collecting configuration diffs for {obj}") + + return redirect(obj.get_absolute_url()) + + +class JobListView(generic.ObjectListView): + queryset = Job.objects.filter( + object_type=ContentType.objects.get(app_label="netbox_config_diff", model="configurationrequest") + ) + filterset = JobFilterSet + filterset_form = JobFilterForm + table = JobTable + actions = ("export", "delete", "bulk_delete") + + +@register_model_view(Substitute) +class SubstituteView(generic.ObjectView): + queryset = Substitute.objects.all() + + +class SubstituteListView(generic.ObjectListView): + queryset = Substitute.objects.prefetch_related("platform_setting", "tags") + filterset = SubstituteFilterSet + filterset_form = SubstituteFilterForm + table = SubstituteTable + + +@register_model_view(Substitute, "edit") +class SubstituteEditView(generic.ObjectEditView): + queryset = Substitute.objects.all() + form = SubstituteForm + + +@register_model_view(Substitute, "delete") +class SubstituteDeleteView(generic.ObjectDeleteView): + queryset = Substitute.objects.all() diff --git a/requirements/base.txt b/requirements/base.txt index 5488b7a..80304ec 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,3 +1,4 @@ netutils==1.5.0 -scrapli[asyncssh]==2023.01.30 -scrapli-community==2023.01.30 +scrapli[asyncssh]==2023.07.30 +scrapli-cfg==2023.07.30 +scrapli-community==2023.07.30 diff --git a/requirements/dev.txt b/requirements/dev.txt index 361a62e..844be90 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,2 +1,2 @@ -black==23.7.0 -ruff==0.0.280 \ No newline at end of file +black==23.10.0 +ruff==0.1.0 diff --git a/tests/factories.py b/tests/factories.py index 59b6d38..408efbb 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -2,6 +2,7 @@ import factory.fuzzy from core.models import DataSource from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Platform, Site +from extras.models import ConfigTemplate from factory.django import DjangoModelFactory from ipam.models import IPAddress @@ -56,6 +57,14 @@ class Meta: model = IPAddress +class ConfigTemplateFactory(DjangoModelFactory): + name = factory.Sequence(lambda n: f"configcontext-{n}") + template_code = factory.fuzzy.FuzzyText() + + class Meta: + model = ConfigTemplate + + class DeviceFactory(DjangoModelFactory): name = factory.Sequence(lambda n: f"device-{n}") site = factory.SubFactory(SiteFactory) @@ -63,6 +72,7 @@ class DeviceFactory(DjangoModelFactory): device_role = factory.SubFactory(DeviceRoleFactory) platform = factory.SubFactory(PlatformFactory) primary_ip4 = factory.SubFactory(IPAddressFactory) + config_template = factory.SubFactory(ConfigTemplateFactory) class Meta: model = Device diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 8dc8a5c..f274ec8 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -133,7 +133,7 @@ def test_devicedataclass_to_scrapli(devicedataclass_data: "DeviceDataClassData") "diff, error, status", [ ("", "asyncio.exceptions.CancelledError", "errored"), - ("there is a diff", "", "failed"), + ("there is a diff", "", "diff"), ("", "", "compliant"), ], ids=["errored", "failed", "compliant"], diff --git a/tests/test_compliance_utils.py b/tests/test_compliance_utils.py index 234ea17..4178a26 100644 --- a/tests/test_compliance_utils.py +++ b/tests/test_compliance_utils.py @@ -36,7 +36,7 @@ ids=["part of line", "full line", "no effect"], ) def test_exclude_lines(regex: str, expected: str) -> None: - assert exclude_lines(ACTUAL_CONFIG, regex) == expected + assert exclude_lines(ACTUAL_CONFIG, regex.splitlines()) == expected @pytest.mark.parametrize(