feat: ✨ start adding LibreNMS Integration with Locations and Devices …

…sync

development/development.env

@@ -112,7 +112,12 @@ IPFABRIC_HOST="https://ipfabric.example.com"
 IPFABRIC_SSL_VERIFY="True"
 IPFABRIC_TIMEOUT=15
 
-NAUTOBOT_SSOT_ENABLE_ITENTIAL="True"
+NAUTOBOT_SSOT_ENABLE_ITENTIAL="False"
 
 NAUTOBOT_SSOT_ENABLE_SLURPIT="False"
-SLURPIT_HOST="https://sandbox.slurpit.io"
+SLURPIT_HOST="https://sandbox.slurpit.io"
+
+NAUTOBOT_SSOT_ENABLE_LIBRENMS="False"
+NAUTOBOT_SSOT_LIBRENMS_SYSTEM_OF_RECORD="LibreNMS"
+NAUTOBOT_SSOT_LIBRENMS_HOSTNAME_FIELD="sysName" # hostname or sysName
+NAUTOBOT_SSOT_LIBRENMS_GEOCODE_API_KEY=""

development/nautobot_config.py

@@ -229,6 +229,7 @@
         "enable_infoblox": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_INFOBLOX")),
         "enable_ipfabric": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_IPFABRIC")),
         "enable_itential": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ITENTIAL")),
+        "enable_librenms": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_LIBRENMS", "false")),
         "enable_meraki": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_MERAKI")),
         "enable_servicenow": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_SERVICENOW")),
         "enable_slurpit": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_SLURPIT")),
@@ -268,6 +269,7 @@
         "infoblox_verify_ssl": is_truthy(os.getenv("NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL", "true")),
         "infoblox_wapi_version": os.getenv("NAUTOBOT_SSOT_INFOBLOX_WAPI_VERSION", "v2.12"),
         "infoblox_network_view": os.getenv("NAUTOBOT_SSOT_INFOBLOX_NETWORK_VIEW", ""),
+        "librenms_geocode_api_key": os.getenv("NAUTOBOT_SSOT_LIBRENMS_GEOCODE_API_KEY", ""),
         "servicenow_instance": os.getenv("SERVICENOW_INSTANCE", ""),
         "servicenow_password": os.getenv("SERVICENOW_PASSWORD", ""),
         "servicenow_username": os.getenv("SERVICENOW_USERNAME", ""),

nautobot_ssot/__init__.py

@@ -102,11 +102,13 @@ class NautobotSSOTAppConfig(NautobotAppConfig):
         "dna_center_show_failures": True,
         "enable_aci": False,
         "enable_aristacv": False,
+        "enable_bootstrap": False,
         "enable_device42": False,
         "enable_dna_center": False,
         "enable_citrix_adm": False,
         "enable_infoblox": False,
         "enable_ipfabric": False,
+        "enable_librenms": False,
         "enable_servicenow": False,
         "enable_slurpit": False,
         "enable_itential": False,

nautobot_ssot/integrations/librenms/constants.py

@@ -0,0 +1,20 @@
+"""Constants for LibreNMS SSoT."""
+
+from django.conf import settings
+
+# Import config vars from nautobot_config.py
+PLUGIN_CFG = settings.PLUGINS_CONFIG["nautobot_ssot"]
+
+librenms_status_map = {
+    1: "Active",
+    2: "Offline",
+    True: "Active",
+    False: "Offline",
+}
+
+os_manufacturer_map = {
+    "ping": "Generic",
+    "linux": "Linux",
+    "routeros": "Mikrotik",
+    "unifi": "Ubiquiti",
+}

nautobot_ssot/integrations/librenms/diffsync/adapters/__init__.py

@@ -0,0 +1 @@
+"""Adapter classes for loading DiffSyncModels with data from LibreNMS or Nautobot."""

nautobot_ssot/integrations/librenms/diffsync/adapters/librenms.py

@@ -0,0 +1,116 @@
+"""Nautobot Ssot Librenms Adapter for LibreNMS SSoT app."""
+
+import os
+
+from diffsync import DiffSync
+from diffsync.exceptions import ObjectNotFound
+from django.core.exceptions import ValidationError
+
+from nautobot_ssot.integrations.librenms.constants import librenms_status_map, os_manufacturer_map
+from nautobot_ssot.integrations.librenms.diffsync.models.librenms import LibrenmsDevice, LibrenmsLocation
+from nautobot_ssot.integrations.librenms.utils import get_city_state_geocode, normalize_gps_coordinates
+from nautobot_ssot.integrations.librenms.utils.librenms import LibreNMSApi
+
+
+class LibrenmsAdapter(DiffSync):
+    """DiffSync adapter for LibreNMS."""
+
+    location = LibrenmsLocation
+    device = LibrenmsDevice
+
+    top_level = ["location", "device"]
+
+    def __init__(self, *args, job=None, sync=None, librenms_api: LibreNMSApi, **kwargs):
+        """Initialize LibreNMS.
+
+        Args:
+            job (object, optional): LibreNMS job. Defaults to None.
+            sync (object, optional): LibreNMS DiffSync. Defaults to None.
+            client (object): LibreNMS API client connection object.
+        """
+        super().__init__(*args, **kwargs)
+        self.job = job
+        self.sync = sync
+        self.lnms_api = librenms_api
+
+    def load_location(self, location: dict):
+        """Load Location objects from LibreNMS into DiffSync models."""
+        self.job.logger.debug(f'Loading LibreNMS Location {location["location"]}')
+
+        try:
+            self.get(self.location, location["location"])
+        except ObjectNotFound:
+            # FIXME: Need to fix false errors when API errors occur with GeoCode API causing models to falsely need updates.
+            _parent = "Unknown"
+            if location["lat"] and location["lng"]:
+                _location_info = get_city_state_geocode(latitude=location["lat"], longitude=location["lng"])
+                _parent = ""
+                if _location_info != "Unknown":
+                    _parent = f'{_location_info["city"]}, {_location_info["state"]}'
+            _latitude = None
+            _longitude = None
+            if location["lat"]:
+                _latitude = normalize_gps_coordinates(location["lat"])
+            if location["lng"]:
+                _longitude = normalize_gps_coordinates(location["lng"])
+            new_location = self.location(
+                name=location["location"],
+                status="Active",
+                location_type="Site",
+                parent=_parent,
+                latitude=_latitude,
+                longitude=_longitude,
+                system_of_record=os.getenv("NAUTOBOT_SSOT_LIBRENMS_SYSTEM_OF_RECORD", "LibreNMS"),
+            )
+            self.add(new_location)
+
+    def load_device(self, device: dict):
+        """Load Device objects from LibreNMS into DiffSync models."""
+        self.job.logger.debug(f'Loading LibreNMS Device {device["sysName"]}')
+
+        try:
+            self.get(self.device, device["sysName"])
+        except ObjectNotFound:
+            if device["disabled"]:
+                _status = "Disabled"
+            else:
+                _status = librenms_status_map[device["status"]]
+            new_device = self.device(
+                name=device[self.hostname_field],
+                device_id=device["device_id"],
+                location=device["location"] if device["location"] is not None else None,
+                role=device["type"] if device["type"] is not None else None,
+                serial_no=device["serial"] if device["serial"] is not None else "",
+                status=_status,
+                manufacturer=(
+                    os_manufacturer_map.get(device["os"])
+                    if os_manufacturer_map.get(device["os"]) is not None
+                    else None
+                ),
+                device_type=device["hardware"] if device["hardware"] is not None else None,
+                platform=device["os"] if device["os"] is not None else None,
+                os_version=device["version"] if device["version"] is not None else None,
+                system_of_record=os.getenv("NAUTOBOT_SSOT_LIBRENMS_SYSTEM_OF_RECORD", "LibreNMS"),
+            )
+            self.add(new_device)
+
+    def load(self):
+        """Load data from LibreNMS into DiffSync models."""
+        self.hostname_field = (
+            os.getenv("NAUTOBOT_SSOT_LIBRENMS_HOSTNAME_FIELD", "sysName")
+            if self.job.hostname_field == "env_var"
+            else self.job.hostname_field or "sysName"
+        )
+
+        # all_devices = self.lnms_api.get_librenms_devices()
+        all_devices = self.lnms_api.get_librenms_devices_from_file()
+        self.job.logger.info(f'Loading {all_devices["count"]} Devices from LibreNMS.')
+
+        # all_locations = self.lnms_api.get_librenms_locations()
+        all_locations = self.lnms_api.get_librenms_locations_from_file()
+        self.job.logger.info(f'Loading {all_locations["count"]} Locations from LibreNMS.')
+
+        for _device in all_devices["devices"]:
+            self.load_device(device=_device)
+        for _location in all_locations["locations"]:
+            self.load_location(location=_location)

nautobot_ssot/integrations/librenms/diffsync/adapters/nautobot.py

@@ -0,0 +1,113 @@
+"""Nautobot Adapter for LibreNMS SSoT app."""
+
+import os
+
+from collections import defaultdict
+from typing import Optional
+
+from diffsync import DiffSync
+from diffsync.enum import DiffSyncModelFlags
+from diffsync.exceptions import ObjectNotFound
+from django.core.exceptions import ValidationError
+from django.db.models import ProtectedError
+from django.db.utils import IntegrityError
+from nautobot.dcim.models import Device as OrmDevice
+from nautobot.dcim.models import Interface as OrmInterface
+from nautobot.dcim.models import Location as OrmLocation
+from nautobot.dcim.models import LocationType as OrmLocationType
+from nautobot.extras.models import Relationship as OrmRelationship
+from nautobot.extras.models import RelationshipAssociation as OrmRelationshipAssociation
+from nautobot.extras.models import Status as OrmStatus
+from nautobot.ipam.models import IPAddress as OrmIPAddress
+from nautobot.ipam.models import IPAddressToInterface as OrmIPAddressToInterface
+from nautobot.ipam.models import Namespace
+from nautobot.ipam.models import Prefix as OrmPrefix
+from nautobot.tenancy.models import Tenant as OrmTenant
+
+from nautobot_ssot.integrations.librenms.utils import check_sor_field, get_sor_field_nautobot_object
+from nautobot_ssot.integrations.librenms.diffsync.models.nautobot import (
+    NautobotDevice,
+    NautobotLocation,
+)
+from nautobot_ssot.jobs.base import DataTarget
+
+
+class NautobotAdapter(DiffSync):
+    """DiffSync adapter for Nautobot."""
+
+    location = NautobotLocation
+    device = NautobotDevice
+
+    top_level = ["location", "device"]
+
+    def __init__(self, *args, job=None, sync=None, **kwargs):
+        """Initialize Nautobot.
+
+        Args:
+            job (object, optional): Nautobot job. Defaults to None.
+            sync (object, optional): Nautobot DiffSync. Defaults to None.
+        """
+        super().__init__(*args, **kwargs)
+        self.job = job
+        self.sync = sync
+
+    def load_location(self):
+        """Load Location objects from Nautobot into DiffSync Models."""
+        for nb_location in OrmLocation.objects.all():
+            self.job.logger.debug(f"Loading Nautobot Location {nb_location}")
+            try:
+                self.get(self.location, nb_location.name)
+            except ObjectNotFound:
+                _parent = None
+                if nb_location.parent is not None:
+                    _parent = nb_location.parent.name
+                new_location = NautobotLocation(
+                    name=nb_location.name,
+                    location_type=nb_location.location_type.name,
+                    parent=_parent,
+                    latitude=nb_location.latitude,
+                    longitude=nb_location.longitude,
+                    status=nb_location.status.name,
+                    system_of_record=get_sor_field_nautobot_object(nb_location),
+                    uuid=nb_location.id,
+                )
+                if not check_sor_field(nb_location):
+                    new_location.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST
+
+                self.add(new_location)
+
+    def load_device(self):
+        """Load Device objects from Nautobot into DiffSync models."""
+        for nb_device in OrmDevice.objects.all():
+            self.job.logger.debug(f"Loading Nautobot Device {nb_device}")
+            try:
+                self.get(self.device, nb_device.name)
+            except ObjectNotFound:
+                try:
+                    _software_version = nb_device.software_version.version
+                except AttributeError:
+                    _software_version = None
+                new_device = NautobotDevice(
+                    name=nb_device.name,
+                    location=nb_device.location.name,
+                    status=nb_device.status.name,
+                    device_type=nb_device.device_type.display.split(f"{nb_device.platform.manufacturer.name}", 1)[
+                        1
+                    ].strip(),
+                    role=nb_device.role.name,
+                    manufacturer=nb_device.platform.manufacturer.name,
+                    platform=nb_device.platform.name,
+                    os_version=_software_version,
+                    serial_no=nb_device.serial,
+                    system_of_record=get_sor_field_nautobot_object(nb_device),
+                    uuid=nb_device.id,
+                )
+                if not check_sor_field(nb_device):
+                    new_device.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST
+
+                self.add(new_device)
+
+    def load(self):
+        """Load data from Nautobot into DiffSync models."""
+        self.load_location()
+        self.load_device()

nautobot_ssot/integrations/librenms/diffsync/models/__init__.py

@@ -0,0 +1 @@
+"""DiffSync models and adapters for the LibreNMS SSoT app."""