Merge pull request #108 from DanSheps/develop

Version bump for 4.2 and add Compliance module

netbox_config_backup/__init__.py

@@ -1,6 +1,12 @@
+from datetime import datetime
 from importlib.metadata import metadata
 
+from django.utils import timezone
+from django_rq import get_queue
+
+from core.choices import JobStatusChoices
 from netbox.plugins import PluginConfig
+from netbox_config_backup.utils.logger import get_logger
 
 metadata = metadata('netbox_config_backup')
 
@@ -14,7 +20,7 @@ class NetboxConfigBackup(PluginConfig):
     author_email = metadata.get('Author-email')
     base_url = 'configbackup'
     min_version = '4.1.0'
-    max_version = '4.1.99'
+    max_version = '4.2.99'
     required_settings = [
         'repository',
         'committer',
@@ -35,9 +41,17 @@ def ready(self, *args, **kwargs):
         if 'rqworker' in sys.argv[1]:
             from netbox import settings
             from netbox_config_backup.jobs.backup import BackupRunner
-            from netbox_config_backup.models import BackupJob, Backup
             frequency = settings.PLUGINS_CONFIG.get('netbox_config_backup', {}).get('frequency') / 60
-            BackupRunner.enqueue_once(interval=frequency)
+            lastjob = BackupRunner.get_jobs().order_by('pk').last()
+
+            if lastjob.status in JobStatusChoices.ENQUEUED_STATE_CHOICES and lastjob.scheduled < timezone.now():
+                BackupRunner.enqueue_once(interval=frequency)
+            elif lastjob.status in JobStatusChoices.TERMINAL_STATE_CHOICES:
+                scheduled = lastjob.created + timezone.timedelta(minutes=frequency)
+                if scheduled < timezone.now():
+                    scheduled = None
+                BackupRunner.enqueue_once(interval=frequency, schedule_at=scheduled)
+
 
 
 config = NetboxConfigBackup

netbox_config_backup/jobs/backup.py

@@ -59,8 +59,17 @@ def clean_stale_jobs(self):
                 job.data.update({'error': 'Process terminated'})
                 job.save()
 
-    def schedule_jobs(self):
-        backups = Backup.objects.filter(status=StatusChoices.STATUS_ACTIVE, device__isnull=False)
+    def schedule_jobs(self, backup=None, device=None):
+        if backup:
+            logging.debug(f'Scheduling backup for backup: {backup}')
+            backups = Backup.objects.filter(pk=backup.pk, status=StatusChoices.STATUS_ACTIVE, device__isnull=False)
+        elif device:
+            logging.debug(f'Scheduling backup for device: {device}')
+            backups = Backup.objects.filter(device=device, status=StatusChoices.STATUS_ACTIVE, device__isnull=False)
+        else:
+            logging.debug(f'Scheduling all backups')
+            backups = Backup.objects.filter(status=StatusChoices.STATUS_ACTIVE, device__isnull=False)
+
         for backup in backups:
             if can_backup(backup):
                 logger.debug(f'Queuing device {backup.device} for backup')
@@ -92,7 +101,7 @@ def schedule_jobs(self):
                     job.save()
 
     def run_processes(self):
-        for job in BackupJob.objects.filter(status=JobStatusChoices.STATUS_SCHEDULED):
+        for job in BackupJob.objects.filter(runner=self.job, status=JobStatusChoices.STATUS_SCHEDULED):
             try:
                 process = self.fork_process(job)
                 process.join(1)
@@ -134,10 +143,10 @@ def handle_processes(self):
                     job.data.update({'error': 'Process terminated'})
                     job.save()
 
-    def run(self, *args, **kwargs):
+    def run(self, backup=None, device=None, *args, **kwargs):
         try:
             self.clean_stale_jobs()
-            self.schedule_jobs()
+            self.schedule_jobs(backup=backup, device=device)
             self.run_processes()
             while(True):
                 self.handle_processes()

netbox_config_backup/management/commands/runbackup.py

@@ -1,28 +1,27 @@
-import uuid
-
 from django.core.management.base import BaseCommand
-from django.db import transaction
-from django.utils import timezone
 
-from netbox_config_backup.models import BackupJob
-from netbox_config_backup.utils.rq import can_backup
+from netbox_config_backup.jobs.backup import BackupRunner
+from netbox_config_backup.models import Backup
 
 
 class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument('--time', dest='time', help="time")
         parser.add_argument('--device', dest='device', help="Device Name")
 
+    def run_backup(self, backup=None):
+        BackupRunner.enqueue(backup=backup, immediate=True)
+
     def handle(self, *args, **options):
-        from netbox_config_backup.models import Backup
         if options['device']:
-            print(f'Running:{options.get("device")}| ')
+            print(f'Running backup for: {options.get("device")}')
             backup = Backup.objects.filter(device__name=options['device']).first()
+            if not backup:
+                backup = Backup.objects.filter(name=options['device']).first()
             if backup:
                 self.run_backup(backup)
             else:
                 raise Exception('Device not found')
         else:
-            for backup in Backup.objects.all():
-                self.run_backup(backup)
+            self.run_backup()
 

netbox_config_backup/models/backups.py

@@ -31,6 +31,7 @@ class Backup(PrimaryModel):
     device = models.ForeignKey(
         to=Device,
         on_delete=models.SET_NULL,
+        related_name='backups',
         blank=True,
         null=True
     )

netbox_config_backup/tables.py

@@ -8,6 +8,9 @@
 class ActionButtonsColumn(tables.TemplateColumn):
     attrs = {'td': {'class': 'text-end text-nowrap noprint min-width'}}
     template_code = """
+    <a href="{% url 'plugins:netbox_config_backup:backup_compliance' backup=record.backup.pk current=record.pk %}" class="btn btn-sm btn-outline-dark" title="View">
+        <i class="mdi mdi-check-all"></i>
+    </a>
     <a href="{% url 'plugins:netbox_config_backup:backup_config' backup=record.backup.pk current=record.pk %}" class="btn btn-sm btn-outline-dark" title="View">
         <i class="mdi mdi-cloud-download"></i>
     </a>

netbox_config_backup/template_content.py

@@ -1,49 +0,0 @@
-import logging
-
-from netbox.plugins import PluginTemplateExtension
-
-from netbox_config_backup.models import Backup, BackupJob, BackupCommitTreeChange
-from netbox_config_backup.tables import BackupsTable
-from netbox_config_backup.utils.backups import get_backup_tables
-from utilities.htmx import htmx_partial
-
-logger = logging.getLogger(f"netbox_config_backup")
-
-
-class DeviceBackups(PluginTemplateExtension):
-    model = 'dcim.device'
-
-    def full_width_page(self):
-        request = self.context.get('request')
-        def build_table(instance):
-            bctc = BackupCommitTreeChange.objects.filter(
-                backup=instance,
-                file__isnull=False
-            )
-            table = BackupsTable(bctc, user=request.user)
-            table.configure(request)
-
-            return table
-
-        device = self.context.get('object', None)
-        devices = Backup.objects.filter(device=device) if device is not None else Backup.objects.none()
-        if devices.count() > 0:
-            instance = devices.first()
-            table = build_table(instance)
-
-            if htmx_partial(request):
-                return self.render('htmx/table.html', extra_context={
-                    'object': instance,
-                    'table': table,
-                    'preferences': {},
-                })
-            return self.render('netbox_config_backup/inc/backup_tables.html', extra_context={
-                'object': instance,
-                'table': table,
-                'preferences': request.user.config,
-            })
-
-        return ''
-
-
-template_extensions = [DeviceBackups]

netbox_config_backup/templates/netbox_config_backup/backups.html

@@ -21,12 +21,14 @@
 
 {% block bulk_edit_controls %}
     {{ block.super }}
-    {% with diff_view=object|viewname:"diff" %}
+    {%  if backup %}
+        {% with diff_view=backup|viewname:"diff" %}
 
-            <button type="submit" name="_diff"
-                    formaction="{% url diff_view backup=object.pk %}?return_url={{ return_url }}"
-                    class="btn btn-warning btn-sm">
-                <i class="mdi mdi-file-compare" aria-hidden="true"></i> Diff Selected (Max 2)
-            </button>
-    {% endwith %}
+                <button type="submit" name="_diff"
+                        formaction="{% url diff_view backup=backup.pk %}?return_url={{ return_url }}"
+                        class="btn btn-warning btn-sm">
+                    <i class="mdi mdi-file-compare" aria-hidden="true"></i> Diff Selected (Max 2)
+                </button>
+        {% endwith %}
+    {% endif %}
 {% endblock bulk_edit_controls %}

netbox_config_backup/templates/netbox_config_backup/compliance.html

@@ -0,0 +1,34 @@
+{% extends 'generic/object.html' %}
+{% load helpers %}
+
+{% block subtitle %}
+  <div class="object-subtitle">
+    <span>{{ current.commit.time }}</span>
+  </div>
+{% endblock %}
+
+{% block content %}
+<div class="row">
+	<div class="col col-md-12">
+        <div class="card">
+            <h5 class="card-header">
+                Configuration Compliance
+            </h5>
+            <div class="card-body">
+                <pre class="change-data">{% for line in diff %}{% spaceless %}
+                    {% if line == '+++' or line == '---' %}
+                    {% elif line|make_list|first == '@' %}
+                        <span style="background-color: #ffdb58; color: black;">{{line}}</span>
+                    {% elif line|make_list|first == '+' %}
+                        <span class="added"> {{line|make_list|slice:'1:'|join:''}}</span>
+                    {% elif line|make_list|first == '-' %}
+                        <span class="removed"> {{line|make_list|slice:'1:'|join:''}}</span>
+                    {% else %}
+                        <span>{{line}}</span>
+                    {% endif %}
+                {% endspaceless %}{% endfor %}</pre>
+            </div>
+        </div>
+	</div>
+</div>
+{% endblock %}

netbox_config_backup/urls.py

@@ -18,6 +18,8 @@
     path('devices/<int:backup>/config/<int:current>/', views.ConfigView.as_view(), name='backup_config'),
     path('devices/<int:backup>/diff/', views.DiffView.as_view(), name='backup_diff'),
     path('devices/<int:backup>/diff/<int:current>/', views.DiffView.as_view(), name='backup_diff'),
+    path('devices/<int:backup>/compliance/', views.ComplianceView.as_view(), name='backup_compliance'),
+    path('devices/<int:backup>/compliance/<int:current>/', views.ComplianceView.as_view(), name='backup_compliance'),
     path('devices/edit/', views.BackupBulkEditView.as_view(), name='backup_bulk_edit'),
     path('devices/delete/', views.BackupBulkDeleteView.as_view(), name='backup_bulk_delete'),
     path('devices/<int:backup>/diff/<int:current>/<int:previous>/', views.DiffView.as_view(), name='backup_diff'),

netbox_config_backup/views.py

@@ -1,12 +1,16 @@
 import logging
 
+from django.contrib import messages
 from django.db import models
 from django.http import Http404
 from django.shortcuts import get_object_or_404, render
 from django.urls import reverse, NoReverseMatch
+from django.utils.translation import gettext as _
 from django.views import View
+from jinja2 import TemplateError
 
 from core.choices import JobStatusChoices
+from dcim.models import Device
 from netbox.views.generic import ObjectDeleteView, ObjectEditView, ObjectView, ObjectListView, ObjectChildrenView, \
     BulkEditView, BulkDeleteView
 from netbox_config_backup.filtersets import BackupFilterSet, BackupsFilterSet, BackupJobFilterSet
@@ -101,13 +105,12 @@ def get_children(self, request, parent):
 
     def get_extra_context(self, request, instance):
         return {
+            'backup': instance,
             'running': bool(request.GET.get('type') == 'running'),
             'startup': bool(request.GET.get('type') == 'startup'),
         }
 
 
-
-
 @register_model_view(Backup, 'edit')
 class BackupEditView(ObjectEditView):
     queryset = Backup.objects.all()
@@ -203,6 +206,73 @@ def get(self, request, backup, current=None):
         })
 
 
+@register_model_view(Backup, 'compliance')
+class ComplianceView(ObjectView):
+    queryset = Backup.objects.all()
+    template_name = 'netbox_config_backup/compliance.html'
+    tab = ViewTab(
+        label='Compliance',
+        weight=500,
+    )
+
+    def get_rendered_config(self, request, backup):
+        instance = backup.device
+        config_template = instance.get_config_template()
+        context_data = instance.get_config_context()
+        context_data.update({'device': instance})
+        try:
+            rendered_config = config_template.render(context=context_data)
+        except TemplateError as e:
+            messages.error(request, _("An error occurred while rendering the template: {error}").format(error=e))
+            rendered_config = ''
+        return rendered_config
+
+    def get_current_backup(self, current, backup):
+        if current:
+            current = get_object_or_404(BackupCommitTreeChange.objects.all(), pk=current)
+        else:
+            current = BackupCommitTreeChange.objects.filter(backup=backup, file__isnull=False).last()
+            if not current:
+                raise Http404(
+                    "No current commit available"
+                )
+        repo = GitBackup()
+        current_sha = current.commit.sha if current.commit is not None else 'HEAD'
+        current_config = repo.read(current.file.path, current_sha)
+
+    def get_diff(self, backup, rendered, current):
+        if backup.device and backup.device.platform.napalm.napalm_driver in ['ios', 'nxos']:
+            differ = Differ(rendered, current)
+            diff = differ.cisco_compare()
+        else:
+            differ = Differ(rendered, current)
+            diff = differ.compare()
+
+        for idx, line in enumerate(diff):
+            diff[idx] = line.rstrip()
+        return diff
+
+
+    def get(self, request, backup, current=None, previous=None):
+        backup = get_object_or_404(Backup.objects.all(), pk=backup)
+
+        diff = ['No rendered configuration', ]
+        rendered_config = None
+        if backup.device and backup.device.get_config_template():
+            rendered_config = self.get_rendered_config(request=request, backup=backup)
+            current_config = self.get_current_backup(backup=backup, current=current)
+            if rendered_config:
+                diff = self.get_diff(backup=backup, rendered=rendered_config, current=current_config)
+
+        return render(request, self.template_name, {
+            'object': backup,
+            'tab': self.tab,
+            'diff': diff,
+            'current': current,
+            'active_tab': 'compliance',
+        })
+
+
 @register_model_view(Backup, 'diff')
 class DiffView(ObjectView):
     queryset = Backup.objects.all()
@@ -282,3 +352,33 @@ def get(self, request, backup, current=None, previous=None):
             'previous': previous,
             'active_tab': 'diff',
         })
+
+
+@register_model_view(Device, name='backups')
+class DeviceBackupsView(ObjectChildrenView):
+    queryset = Device.objects.all()
+
+    template_name = 'netbox_config_backup/backups.html'
+    child_model = BackupCommitTreeChange
+    table = BackupsTable
+    filterset = BackupsFilterSet
+    actions = {
+        'config': {'view'},
+        'diff': {'view'},
+        'bulk_diff': {'view'}
+    }
+    tab = ViewTab(
+        label='Backups',
+        weight=100,
+        badge=lambda obj: BackupCommitTreeChange.objects.filter(backup__device=obj, file__isnull=False).count(),
+    )
+
+    def get_children(self, request, parent):
+        return self.child_model.objects.filter(backup__device=parent, file__isnull=False)
+
+    def get_extra_context(self, request, instance):
+        return {
+            'backup': instance.backups.filter(status='active').last(),
+            'running': bool(request.GET.get('type') == 'running'),
+            'startup': bool(request.GET.get('type') == 'startup'),
+        }