Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial working version of nautobot-ssot plugin #1

Merged
merged 42 commits into from
Jul 2, 2021
Merged
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
e29a643
Initial content creation
glennmatthews May 17, 2021
04c867c
More progress towards a basic functioning UI
glennmatthews May 24, 2021
8899de3
Create nautobot_data_sync_servicenow
glennmatthews May 28, 2021
3f367f5
Renames
glennmatthews May 28, 2021
e144eea
Add dashboard view, add source/target/start_time fields to Sync model…
glennmatthews Jun 4, 2021
7a47c4a
Extract ServiceNow plugin to its own repository
glennmatthews Jun 7, 2021
c304c6a
Some refactoring and point fixes
glennmatthews Jun 7, 2021
198a7fe
Log traceback if exception occurs during sync
glennmatthews Jun 8, 2021
6c50e2f
Rename changed_object field to synced_object to be more accurate
glennmatthews Jun 10, 2021
697717e
Breadcrumb fixup
glennmatthews Jun 10, 2021
a39a034
Improve datetime rendering in str(Sync)
glennmatthews Jun 10, 2021
9e6f50b
Improve sync detail view
glennmatthews Jun 10, 2021
64f073f
Various content
glennmatthews Jun 14, 2021
8b108c3
Refactor to more closely align with Jobs core functionality
glennmatthews Jun 15, 2021
1bcf8f2
Temporarily add dependency on my PR branch
glennmatthews Jun 15, 2021
3b9a5de
Initial very rough version of data_source_target view
glennmatthews Jun 15, 2021
81b61c9
Refine per-data-source/target detail view
glennmatthews Jun 16, 2021
ec0e365
Fix bug with null object_repr, refine UI
glennmatthews Jun 17, 2021
c1d0ea5
Improved logging; a few UI tweaks in progress
glennmatthews Jun 17, 2021
7e2e03c
Update temporary Nautobot dependency now that nautobot/nautobot#576 i…
glennmatthews Jun 18, 2021
db5bcf3
Combine synced_object and object_repr columns into one
glennmatthews Jun 18, 2021
1ec71ab
Diff rendering, first version
glennmatthews Jun 18, 2021
174d30f
Avoid an error if the related_object doesn't have a class_path
glennmatthews Jun 18, 2021
0a077e7
Blacken
glennmatthews Jun 18, 2021
22dfb43
Linting
glennmatthews Jun 18, 2021
2f2521f
Regenerate migrations and remove ChangeLoggedModel from Sync
glennmatthews Jun 18, 2021
8aba02b
Fix CI?
glennmatthews Jun 18, 2021
3e2f836
Fix sync_logentries view
glennmatthews Jun 18, 2021
c70c89b
Another CI fix?
glennmatthews Jun 18, 2021
d21477d
Another attempt at fixing CI
glennmatthews Jun 18, 2021
0a5b269
Various UI refinements
glennmatthews Jun 23, 2021
97eb871
Update nautobot dependency, add some docs
glennmatthews Jun 24, 2021
658bd72
Review comments
glennmatthews Jun 25, 2021
076b8d5
Rework 'Statistics' table for readability
glennmatthews Jun 25, 2021
1a782f0
Add config toggle to hide example jobs
glennmatthews Jun 28, 2021
6bbc7ec
Update example jobs to be more realistic, using DiffSync
glennmatthews Jun 28, 2021
39a5600
Fix incorrect filtering of data in dashboard view
glennmatthews Jun 28, 2021
f7d04df
Linting
glennmatthews Jun 28, 2021
d105111
Remove ObjectChange linkage from SyncLogEntry and lookup_object() as …
glennmatthews Jun 28, 2021
ec17c25
Add some unit test coverage
glennmatthews Jun 28, 2021
c0f3ee8
Address review comment
glennmatthews Jun 30, 2021
220b6e3
Update docs and add screenshots
glennmatthews Jul 2, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Linting
  • Loading branch information
glennmatthews committed Jun 18, 2021
commit 22dfb438c2a7e4b4e145b0ce7acc36d1cc1a7d5d
7 changes: 6 additions & 1 deletion nautobot_ssot/filters.py
Original file line number Diff line number Diff line change
@@ -12,6 +12,8 @@ class SyncFilter(BaseFilterSet):
"""Filter capabilities for SyncOverview instances."""

class Meta:
"""Metaclass attributes of SyncFilter."""

model = Sync
fields = ["dry_run", "job_result"]

@@ -22,10 +24,13 @@ class SyncLogEntryFilter(BaseFilterSet):
q = django_filters.CharFilter(method="search", label="Search")

class Meta:
"""Metaclass attributes of SyncLogEntryFilter."""

model = SyncLogEntry
fields = ["sync", "action", "status", "synced_object_type"]

def search(self, queryset, name, value):
def search(self, queryset, _name, value): # pylint: disable=no-self-use
"""String search of SyncLogEntry records."""
if not value.strip():
return queryset
return queryset.filter(Q(diff__icontains=value) | Q(message_icontains=value))
4 changes: 4 additions & 0 deletions nautobot_ssot/forms.py
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@ class SyncFilterForm(BootstrapMixin, forms.ModelForm):
dry_run = forms.ChoiceField(choices=BOOLEAN_WITH_BLANK_CHOICES, required=False)

class Meta:
"""Metaclass attributes of SyncFilterForm."""

model = Sync
fields = ["dry_run"]

@@ -27,6 +29,8 @@ class SyncLogEntryFilterForm(BootstrapMixin, forms.ModelForm):
status = forms.ChoiceField(choices=add_blank_choice(SyncLogEntryStatusChoices), required=False)

class Meta:
"""Metaclass attributes of SyncLogEntryFilterForm."""

model = SyncLogEntry
fields = ["sync", "action", "status"]

2 changes: 2 additions & 0 deletions nautobot_ssot/jobs/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Plugin provision of Nautobot Job subclasses."""

from nautobot.extras.jobs import get_jobs

from .base import DataSource, DataTarget
22 changes: 15 additions & 7 deletions nautobot_ssot/jobs/base.py
Original file line number Diff line number Diff line change
@@ -6,6 +6,9 @@
from django.utils import timezone
from django.utils.functional import classproperty

# pylint-django doesn't understand classproperty, and complains unnecessarily. We disable this specific warning:
# pylint: disable=no-self-argument

import structlog

from nautobot.extras.jobs import BaseJob, BooleanVar
@@ -41,9 +44,9 @@ def sync_data(self):
- self.sync (Sync instance tracking this job execution)
- self.job_result (as per Job API)
"""
pass
pass # pylint: disable=unnecessary-pass

def lookup_object(self, model_name, unique_id):
def lookup_object(self, model_name, unique_id): # pylint: disable=no-self-use,unused-argument
"""Look up the Nautobot record and associated ObjectChange, if any, identified by the args.

Optional helper method used to build more detailed/accurate SyncLogEntry records from DiffSync logs.
@@ -75,7 +78,7 @@ def config_information(cls):
"""
return {}

def sync_log(
def sync_log( # pylint: disable=too-many-arguments
self,
action,
status,
@@ -121,14 +124,14 @@ def _structlog_to_sync_log_entry(self, _logger, _log_method, event_dict):

@classmethod
def _get_vars(cls):
"""Extend Job._get_vars() to include `dry_run` variable.
"""Extend Job._get_vars to include `dry_run` variable.

Workaround for https://github.com/netbox-community/netbox/issues/5529
"""
vars = super()._get_vars()
got_vars = super()._get_vars()
if hasattr(cls, "dry_run"):
vars["dry_run"] = cls.dry_run
return vars
got_vars["dry_run"] = cls.dry_run
return got_vars

def as_form(self, data=None, files=None, initial=None):
"""Render this instance as a Django form for user inputs, including a "Dry run" field."""
@@ -161,6 +164,7 @@ def data_target_icon(cls):

def run(self, data, commit):
"""Job entry point from Nautobot - do not override!"""
# pylint: disable=attribute-defined-outside-init
self.sync = Sync.objects.create(
source=self.data_source,
target=self.data_target,
@@ -192,10 +196,12 @@ class DataSource(DataSyncBaseJob):

@classproperty
def data_target(cls):
"""For a DataSource this is always Nautobot."""
return "Nautobot"

@classproperty
def data_target_icon(cls):
"""For a DataSource this is always the Nautobot logo."""
return static("img/nautobot_logo.png")


@@ -206,8 +212,10 @@ class DataTarget(DataSyncBaseJob):

@classproperty
def data_source(cls):
"""For a DataTarget this is always Nautobot."""
return "Nautobot"

@classproperty
def data_source_icon(cls):
"""For a DataTarget this is always the Nautobot logo."""
return static("img/nautobot_logo.png")
8 changes: 8 additions & 0 deletions nautobot_ssot/jobs/examples.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Sample data-source and data-target Jobs."""

from django.urls import reverse

from nautobot.dcim.models import Site
@@ -13,12 +15,15 @@ class ExampleDataSource(DataSource, Job):
site_slug = StringVar(description="Site to create or update", default="")

class Meta:
"""Metaclass attributes of ExampleDataSource."""

name = "Example Data Source"
description = "An example of a 'data source' Job for loading data into Nautobot from elsewhere."
data_source = "Dummy Data"

@classmethod
def data_mappings(cls):
"""This Job maps a site slug to a Site object."""
return (DataMapping("site slug", None, "Site", reverse("dcim:site_list")),)

def sync_data(self):
@@ -42,12 +47,15 @@ class ExampleDataTarget(DataTarget, Job):
site_slug = StringVar(description="Site to sync to an imaginary data target", default="")

class Meta:
"""Metaclass attributes of ExampleDataTarget."""

name = "Example Data Target"
description = "An example of a 'data target' Job for loading data from Nautobot into elsewhere."
data_target = "Dummy Data"

@classmethod
def data_mappings(cls):
"""This Job maps a Site object to a site slug."""
return (DataMapping("Site", reverse("dcim:site_list"), "site slug", None),)

def sync_data(self):
6 changes: 6 additions & 0 deletions nautobot_ssot/models.py
Original file line number Diff line number Diff line change
@@ -52,12 +52,16 @@ class Sync(BaseModel, ChangeLoggedModel, CustomFieldModel, RelationshipModel):
job_result = models.ForeignKey(to=JobResult, on_delete=models.PROTECT, blank=True, null=True)

class Meta:
"""Metaclass attributes of Sync model."""

ordering = ["start_time"]

def __str__(self):
"""String representation of a Sync instance."""
return f"{self.source} → {self.target}, {date_format(self.start_time, format=settings.SHORT_DATETIME_FORMAT)}"

def get_absolute_url(self):
"""Get the detail-view URL for this instance."""
return reverse("plugins:nautobot_ssot:sync", kwargs={"pk": self.pk})

@classmethod
@@ -147,6 +151,8 @@ class SyncLogEntry(BaseModel):
message = models.CharField(max_length=511, blank=True)

class Meta:
"""Metaclass attributes of SyncLogEntry."""

verbose_name_plural = "sync log entries"
ordering = ["sync", "timestamp"]

3 changes: 1 addition & 2 deletions nautobot_ssot/navigation.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""Plugin additions to the Nautobot navigation menu."""

from nautobot.extras.plugins import PluginMenuItem, PluginMenuButton
from nautobot.utilities.choices import ButtonColorChoices
from nautobot.extras.plugins import PluginMenuItem


menu_items = (
8 changes: 8 additions & 0 deletions nautobot_ssot/tables.py
Original file line number Diff line number Diff line change
@@ -45,6 +45,8 @@ class DashboardTable(BaseTable):
dry_run = TemplateColumn(template_code=DRY_RUN_LABEL, verbose_name="Type")

class Meta(BaseTable.Meta):
"""Metaclass attributes of DashboardTable."""

model = Sync
fields = ["source", "target", "start_time", "status", "dry_run"]
order_by = ["-start_time"]
@@ -99,6 +101,8 @@ class SyncTable(BaseTable):
)

class Meta(BaseTable.Meta):
"""Metaclass attributes of SyncTable."""

model = Sync
fields = (
"pk",
@@ -137,6 +141,8 @@ class SyncTableSingleSourceOrTarget(SyncTable):
"""Subclass of SyncTable with fewer default columns."""

class Meta(SyncTable.Meta):
"""Metaclass attributes of SyncTableSingleSourceOrTarget."""

default_columns = (
"start_time",
"status",
@@ -178,6 +184,8 @@ class SyncLogEntryTable(BaseTable):
object_change = LinkColumn()

class Meta(BaseTable.Meta):
"""Metaclass attributes of SyncLogEntryTable."""

model = SyncLogEntry
fields = (
"pk",
5 changes: 5 additions & 0 deletions nautobot_ssot/template_content.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
"""Plugin template content extensions of base Nautobot views."""

from django.urls import reverse

from nautobot.extras.plugins import PluginTemplateExtension

from nautobot_ssot.models import Sync

# pylint: disable=abstract-method


class JobResultSyncLink(PluginTemplateExtension):
"""Add button linking to Sync data for relevant JobResults."""

model = "extras.jobresult"

def buttons(self):
"""Inject a custom button into the JobResult detail view, if applicable."""
try:
sync = Sync.objects.get(job_result=self.context["object"])
return f"""
3 changes: 3 additions & 0 deletions nautobot_ssot/templatetags/dashboard_helpers.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Helper templatetag for use with the "dashboard" UI view."""

import logging

from django import template
@@ -11,6 +13,7 @@

@register.inclusion_tag("nautobot_ssot/templatetags/dashboard_data.html")
def dashboard_data(sync_worker_class, queryset, kind="source"):
"""Render data about the sync history of a specific data-source or data-target."""
if kind == "source":
records = queryset.filter(source=sync_worker_class.name).order_by("-start_time")
else:
35 changes: 29 additions & 6 deletions nautobot_ssot/templatetags/render_diff.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,37 @@
"""Template tag for rendering a DiffSync diff dictionary in a more human-readable form."""

from django import template
from django.utils.safestring import mark_safe
from django.utils.html import format_html


register = template.Library()


def render_diff_recursive(diff):
"""Recursively render a DiffSync diff dictionary representation to nested list elements.

Example:
{
location:
ams:
+: {name: "Amsterdam"}
-: {name: "amsterdam"}
device:
ams01:
+: {}
}

would render as:

* location
! ams
+ name: Amsterdam
- name: amsterdam
* device
+ ams01
"""
result = ""
for type, children in diff.items():
for record_type, children in diff.items():
child_result = ""
for child, child_diffs in children.items():
if "+" in child_diffs and "-" not in child_diffs:
@@ -29,9 +53,8 @@ def render_diff_recursive(diff):
if child_diffs:
child_result += render_diff_recursive(child_diffs)

child_result += f"</ul></li>"
child_result = f"<ul>{child_result}</ul>"
result += f"<li>{type}{child_result}</li>"
child_result += "</ul></li>"
result += f"<li>{record_type}<ul>{child_result}</ul></li>"
return result


@@ -40,4 +63,4 @@ def render_diff(diff):
"""Render a DiffSync diff dict to HTML."""
result = f"<ul>{render_diff_recursive(diff)}</ul>"

return mark_safe(result)
return format_html(result)
2 changes: 2 additions & 0 deletions nautobot_ssot/templatetags/shorter_timedelta.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Simple template tag to render a shorter representation of a timedelta object."""

from django import template
from django.utils.html import format_html

10 changes: 9 additions & 1 deletion nautobot_ssot/views.py
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@ class DashboardView(ObjectListView):
template_name = "nautobot_ssot/dashboard.html"

def extra_context(self):
"""Extend the view context with additional details."""
data_sources, data_targets = get_data_jobs()
context = {
"queryset": self.queryset,
@@ -55,9 +56,11 @@ class DataSourceTargetView(ContentTypePermissionRequiredMixin, View):
"""Detail view of a given Data Source or Data Target Job."""

def get_required_permission(self):
"""Permissions required to access this view."""
return "extras.view_job"

def get(self, request, class_path):
"""HTTP GET request handler."""
job_class = get_job(class_path)
if not job_class or not issubclass(job_class, (DataSource, DataTarget)):
raise Http404
@@ -87,6 +90,7 @@ class SyncListView(ObjectListView):
template_name = "nautobot_ssot/history.html"

def extra_context(self):
"""Extend the view context with additional information."""
data_sources, data_targets = get_data_jobs()
return {
"data_sources": data_sources,
@@ -143,14 +147,18 @@ class SyncLogEntriesView(ObjectListView):
action_buttons = []
template_name = "nautobot_ssot/sync_logentries.html"

def get(self, request, pk):
def get(self, request, pk): # pylint: disable=arguments-differ
"""HTTP GET request handler."""
instance = get_object_or_404(Sync.objects.all(), pk=pk)
self.queryset = SyncLogEntry.objects.filter(sync=instance)

return super().get(request)


class SyncChangeLogView(ObjectChangeLogView):
"""View for monitoring the changelog of a Sync object."""

# TODO: remove this view, sync should not be a changeloggedmodel
base_template = "nautobot_ssot/sync_header.html"


2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -79,9 +79,11 @@ no-docstring-rgx="^(_|test_|Meta$)"
[tool.pylint.messages_control]
# Line length is enforced by Black, so pylint doesn't need to check it.
# Pylint and Black disagree about how to format multi-line arrays; Black wins.
# "too-few-public-methods" is just plain noise.
disable = """,
line-too-long,
bad-continuation,
too-few-public-methods
"""

[tool.pylint.miscellaneous]
Loading