Merge pull request #674 from nautobot/nautobot_to_librenms

Nautobot to LibreNMS job

changes/672.added

@@ -0,0 +1 @@
+Added job LibreNMSDataTarget job to sync data from Nautobot to LibreNMS.

docs/user/integrations/librenms.md

@@ -2,34 +2,55 @@
 
 ## Process
 
-### LibreNMS as DataSource
-
-The LibreNMS SSoT integration is built as part of the [Nautobot Single Source of Truth (SSoT)](https://github.com/nautobot/nautobot-app-ssot) app. the SSoT app enables Nautobot to be the aggregation point for data coming from multiple systems of record (SoR).
-
-#### Job Options
+### Shared Job Options
 
 - Debug: Additional Logging
 - Librenms Server: External integration object pointing to the required LibreNMS instance.
 - hostname_field: Which LibreNMS field to use as the hostname in Nautobot. sysName or hostanme.
-- sync_location_parents: Whether to lookup City and State to add parent locations for geo locations.
+- sync_locations: Whether to sync locations from Nautobot to LibreNMS.
+- location_type: This is used to filter which locations are synced to LibreNMS. This should be the Location Type that actually has devices assigned. For example, Site. Since LibreNMS does not support nested locations.
 - tenant: This is used as a filter for objects synced with Nautobot and LibreNMS. This can be used to sync multiple LibreNMS instances into different tenants, like in an MSP environment. This affects which devices are loaded from Nautobot during the sync. It does not affect which devices are loaded from LibreNMS
 
-From LibreNMS into Nautobot, the app synchronizes devices, their interfaces, associated IP addresses, and Locations. Here is a table showing the data mappings when syncing from LibreNMS.
+
+### LibreNMS as DataSource
+
+The LibreNMS SSoT integration is built as part of the [Nautobot Single Source of Truth (SSoT)](https://github.com/nautobot/nautobot-app-ssot) app. the SSoT app enables Nautobot to be the aggregation point for data coming from multiple systems of record (SoR).
+
+#### Job Specific Options
+
+- load_type: Whether to load data from a local fixture file or from the External Integration API. File is only used for testing or trying out the integration without a connection to a LibreNMS instance.
+
+From LibreNMS into Nautobot, the app synchronizes devices, and Locations. Here is a table showing the data mappings when syncing from LibreNMS to Nautobot.
 
 | LibreNMS objects        | Nautobot objects             |
 | ----------------------- | ---------------------------- |
 | geo location            | Location                     |
 | device                  | Device                       |
-| interface               | Interface                    |
+| interface               | Interface `**`               |
 | device os               | Platform/Manufacturer `*`    |
 | os version              | Software/SoftwareImage       |
-| ip address              | IPAddress                    |
+| ip address              | IPAddress `**`               |
 | hardware                | DeviceType                   |
 
 
-`*` Device OS from LibreNMS is not standardized and therefore there is a mapping that can be updated in the `constants.py` file for the integration as more device manufacturers and platforms need to be added.
+`*` Device OS from LibreNMS is not standardized and therefore there is a mapping that can be updated in the `constants.py` file for the integration as more device manufacturers and platforms need to be added. If new device manufacturers and platforms are added, open an issue or PR to add them.
+`**` Not yet implemented, but planned for the future.
 
 ### LibreNMS as DataTarget
 
-Not yet implemented.
+This is a job that can be used to sync data from Nautobot to LibreNMS. 
+
+#### Job Specific Options
+
+- force_add: Whether to force add devices to LibreNMS. This will bypass the ICMP check. Will not work correctly until SNMP credential support is added to the LibreNMSDataTarget job.
+- ping_fallback: Whether to add device as ping-only if device is not reachable via SNMP.
+
+From Nautobot into LibreNMS, the app synchronizes devices, and Locations. Here is a table showing the data mappings when syncing from Nautobot to LibreNMS.
+
+| Nautobot objects             | LibreNMS objects        |
+| ---------------------------- | ----------------------- |
+| Device                       | device `*`              |
+| Location                     | geo location `**`       |
 
+`*` Devices in Nautobot must have a primary IP address set for them to be added to LibreNMS.
+`**` Locations must have GPS coordinates set for them to be added to LibreNMS.

nautobot_ssot/integrations/librenms/diffsync/adapters/librenms.py

@@ -17,6 +17,7 @@
     LibrenmsLocation,
 )
 from nautobot_ssot.integrations.librenms.utils import (
+    normalize_device_hostname,
     normalize_gps_coordinates,
 )
 from nautobot_ssot.integrations.librenms.utils.librenms import LibreNMSApi
@@ -81,7 +82,7 @@ def load_device(self, device: dict):
                 else:
                     _status = librenms_status_map[device["status"]]
                 new_device = self.device(
-                    name=device[self.hostname_field],
+                    name=normalize_device_hostname(device[self.hostname_field]),
                     device_id=device["device_id"],
                     location=(device["location"] if device["location"] is not None else "Unknown"),
                     role=device["type"] if device["type"] is not None else None,
@@ -95,6 +96,7 @@ def load_device(self, device: dict):
                     device_type=(device["hardware"] if device["hardware"] is not None else "Unknown"),
                     platform=device["os"] if device["os"] is not None else "Unknown",
                     os_version=(device["version"] if device["version"] is not None else "Unknown"),
+                    ip_address=device["ip"],
                     system_of_record=os.getenv("NAUTOBOT_SSOT_LIBRENMS_SYSTEM_OF_RECORD", "LibreNMS"),
                 )
                 self.add(new_device)

nautobot_ssot/integrations/librenms/diffsync/adapters/nautobot.py

@@ -13,7 +13,11 @@
     NautobotDevice,
     NautobotLocation,
 )
-from nautobot_ssot.integrations.librenms.utils import check_sor_field, get_sor_field_nautobot_object
+from nautobot_ssot.integrations.librenms.utils import (
+    check_sor_field,
+    get_sor_field_nautobot_object,
+    normalize_device_hostname,
+)
 
 
 class NautobotAdapter(DiffSync):
@@ -80,11 +84,15 @@ def load_device(self):
                     _software_version = nb_device.software_version.version
                 except AttributeError:
                     _software_version = None
+                try:
+                    _ip_address = nb_device.primary_ip.host
+                except AttributeError:
+                    _ip_address = None
                 _device_id = None
                 if nb_device.custom_field_data.get("librenms_device_id"):
                     _device_id = nb_device.custom_field_data.get("librenms_device_id")
                 new_device = NautobotDevice(
-                    name=nb_device.name,
+                    name=normalize_device_hostname(nb_device.name),
                     device_id=_device_id,
                     location=nb_device.location.name,
                     status=nb_device.status.name,
@@ -94,6 +102,7 @@ def load_device(self):
                     platform=nb_device.platform.name,
                     os_version=_software_version,
                     serial_no=nb_device.serial,
+                    ip_address=_ip_address,
                     system_of_record=get_sor_field_nautobot_object(nb_device),
                     uuid=nb_device.id,
                 )

nautobot_ssot/integrations/librenms/diffsync/models/base.py

@@ -61,6 +61,7 @@ class Device(DiffSyncModel):
     tenant: Optional[str] = None
     status: str
     device_type: str
+    ip_address: Optional[str] = None
     role: Optional[str] = None
     manufacturer: str
     platform: Optional[str] = None

nautobot_ssot/integrations/librenms/diffsync/models/librenms.py

@@ -1,18 +1,45 @@
 """Nautobot Ssot Librenms DiffSync models for Nautobot Ssot Librenms SSoT."""
 
+from nautobot.dcim.models import Device as NautobotDevice
+
 from nautobot_ssot.integrations.librenms.diffsync.models.base import Device, Location
 
 
 class LibrenmsLocation(Location):
     """LibreNMS implementation of Location DiffSync model."""
 
     @classmethod
-    def create(cls, diffsync, ids, attrs):
+    def create(cls, adapter, ids, attrs):
         """Create Location in LibreNMS from LibrenmsLocation object."""
-        return super().create(diffsync=diffsync, ids=ids, attrs=attrs)
+        if attrs["status"] == "Active" or attrs["status"] == "Staged":
+            if attrs["latitude"] and attrs["longitude"]:
+                location = {
+                    "location": ids["name"],
+                    "lat": attrs["latitude"],
+                    "lng": attrs["longitude"],
+                }
+                adapter.job.logger.info(f"Creating location in LibreNMS: {location['location']}")
+                adapter.lnms_api.create_librenms_location(location)
+            else:
+                adapter.job.logger.warning(
+                    f"Skipping location in LibreNMS: {ids['name']}. Latitude or Longitude is not set, which LibreNMS requires."
+                )
+        else:
+            if adapter.job.debug:
+                adapter.job.logger.debug(
+                    f"Skipping location in LibreNMS: {ids['name']}. Status is not Active or Staged."
+                )
+        return super().create(adapter=adapter, ids=ids, attrs=attrs)
 
     def update(self, attrs):
         """Update Location in LibreNMS from LibrenmsLocation object."""
+        if "latitude" in attrs or "longitude" in attrs:
+            location = {
+                "lat": attrs["latitude"],
+                "lng": attrs["longitude"],
+            }
+            self.adapter.job.logger.info(f"Updating location in LibreNMS: {location}")
+            self.adapter.lnms_api.update_librenms_location(location)
         return super().update(attrs)
 
     def delete(self):
@@ -21,12 +48,41 @@ def delete(self):
 
 
 class LibrenmsDevice(Device):
-    """LibreNMS implementation of Device DiffSync model."""
+    """LibreNMS implementation of Device adapter model."""
 
     @classmethod
-    def create(cls, diffsync, ids, attrs):
+    def create(cls, adapter, ids, attrs):
         """Create Device in LibreNMS from LibrenmsDevice object."""
-        return super().create(diffsync=diffsync, ids=ids, attrs=attrs)
+        if attrs["status"] == "Active" or attrs["status"] == "Staged":
+            device_data = adapter.job.source_adapter.dict()["device"][ids["name"]]
+            if device_data.get("ip_address"):
+                device = {
+                    "hostname": device_data["ip_address"],
+                    "display": ids["name"],
+                    "location": attrs["location"],
+                }
+                if adapter.job.force_add:
+                    device["force_add"] = True
+                if adapter.job.ping_fallback:
+                    device["ping_fallback"] = True
+                adapter.job.logger.info(f"Creating device in LibreNMS: {device['hostname']}")
+                response = adapter.lnms_api.create_librenms_device(device)
+                if response.get("status") == "error":
+                    adapter.job.logger.error(f"Error creating device in LibreNMS: {response['message']}")
+                elif response.get("status") == "ok" and response.get("devices"):
+                    # Get the device ID from the first device in the devices array
+                    librenms_device_id = response["devices"][0]["device_id"]
+                    nautobot_device = NautobotDevice.objects.get(name=ids["name"])
+                    nautobot_device.custom_field_data["librenms_device_id"] = librenms_device_id
+                    nautobot_device.save()
+            else:
+                if adapter.job.debug:
+                    adapter.job.logger.debug(
+                        f"Skipping device in LibreNMS: {ids['name']}. No Primary IP address found."
+                    )
+        else:
+            adapter.job.logger.info(f"Skipping device in LibreNMS: {ids['name']}. Status is not Active or Staged.")
+        return super().create(adapter=adapter, ids=ids, attrs=attrs)
 
     def update(self, attrs):
         """Update Device in LibreNMS from LibrenmsDevice object."""

nautobot_ssot/integrations/librenms/diffsync/models/nautobot.py

@@ -67,12 +67,20 @@ def create(cls, adapter, ids, attrs):
         if adapter.job.debug:
             adapter.job.logger.debug(f'Creating Nautobot Location {ids["name"]}')
 
+        try:
+            _location_type = LocationType.objects.get(id=adapter.job.location_type.id)
+        except LocationType.DoesNotExist:
+            adapter.job.logger.warning(
+                f"Location Type {adapter.job.location_type} does not exist. Using default Site Location Type."
+            )
+            _location_type = LocationType.objects.get(name="Site")
+
         new_location = ORMLocation(
             name=ids["name"],
             latitude=attrs["latitude"],
             longitude=attrs["longitude"],
             status=Status.objects.get(name=attrs["status"]),
-            location_type=LocationType.objects.get(name="Site"),
+            location_type=_location_type,
         )
         if adapter.tenant:
             new_location.tenant = adapter.tenant

nautobot_ssot/integrations/librenms/jobs.py

@@ -5,6 +5,7 @@
 from django.templatetags.static import static
 from nautobot.apps.jobs import BooleanVar, ChoiceVar, ObjectVar
 from nautobot.core.celery import register_jobs
+from nautobot.dcim.models import LocationType
 from nautobot.extras.choices import (
     SecretsGroupAccessTypeChoices,
     SecretsGroupSecretTypeChoices,
@@ -49,10 +50,19 @@ class LibrenmsDataSource(DataSource):
         default="api",
     )
     sync_locations = BooleanVar(description="Whether to Sync Locations from LibreNMS to Nautobot.", default=False)
+    location_type = ObjectVar(
+        model=LocationType,
+        queryset=LocationType.objects.all(),
+        query_params={"content_types": "dcim.device"},
+        display_field="name",
+        required=False,
+        label="Location Type",
+        description="Location Type to use for syncing locations to LibreNMS. This should be the Location Type that actually has devices assigned. For example, Site.",
+    )
     tenant = ObjectVar(
         model=Tenant,
         queryset=Tenant.objects.all(),
-        description="Tenant to limit loading devices when syncing multiple LibreNMS Instances",
+        description="Tenant to filter loaded information from Nautobot when syncing multiple LibreNMS Instances",
         display_field="display",
         label="Tenant Filter",
         required=False,
@@ -81,8 +91,8 @@ def config_information(cls):
     def data_mappings(cls):
         """List describing the data mappings involved in this DataSource."""
         return (
-            DataMapping("Location", "", "Location", "dcim.location"),
-            DataMapping("DeviceGroup", "", "Tag", "extras.tags"),
+            DataMapping("Geo Location", "", "Location", "dcim.location"),
+            DataMapping("Device Group", "", "Tag", "extras.tags"),
             DataMapping("Device", "", "Device", "dcim.device"),
             DataMapping("Port", "", "Interface", "dcim.interfaces"),
             DataMapping("IP", "", "IPAddress", "ipam.ip_address"),
@@ -127,6 +137,7 @@ def run(
         librenms_server,
         hostname_field,
         sync_locations,
+        location_type,
         tenant,
         load_type,
         *args,
@@ -137,6 +148,7 @@ def run(
         self.hostname_field = hostname_field
         self.load_type = load_type
         self.sync_locations = sync_locations
+        self.location_type = location_type
         self.tenant = tenant
         self.debug = debug
         self.dryrun = dryrun
@@ -154,6 +166,28 @@ class LibrenmsDataTarget(DataTarget):
         required=True,
         label="LibreNMS Instance",
     )
+    force_add = BooleanVar(description="Force add devices to LibreNMS (bypass ICMP check)", default=False)
+    ping_fallback = BooleanVar(description="Fallback to ICMP check if device is not reachable via SNMP", default=False)
+    sync_locations = BooleanVar(description="Whether to Sync Locations from Nautobot to LibreNMS.", default=False)
+    location_type = ObjectVar(
+        model=LocationType,
+        queryset=LocationType.objects.all(),
+        query_params={"content_types": "dcim.device"},
+        display_field="name",
+        required=False,
+        label="Location Type",
+        description="Location Type to use for syncing locations to LibreNMS. This should be the Location Type that actually has devices assigned. For example, Site.",
+    )
+    hostname_field = ""
+    load_type = ""
+    tenant = ObjectVar(
+        model=Tenant,
+        queryset=Tenant.objects.all(),
+        description="Tenant to filter loaded information from Nautobot when syncing multiple LibreNMS Instances",
+        display_field="display",
+        label="Tenant Filter",
+        required=False,
+    )
     debug = BooleanVar(description="Enable for more verbose debug logging", default=False)
 
     class Meta:  # pylint: disable=too-few-public-methods
@@ -164,6 +198,7 @@ class Meta:  # pylint: disable=too-few-public-methods
         data_target = "LibreNMS"
         description = "Sync information from Nautobot to LibreNMS"
         data_target_icon = static("nautobot_ssot_librenms/librenms.svg")
+        has_sensitive_variables = False
 
     @classmethod
     def config_information(cls):
@@ -173,7 +208,11 @@ def config_information(cls):
     @classmethod
     def data_mappings(cls):
         """List describing the data mappings involved in this DataSource."""
-        return ()
+        return (
+            DataMapping("dcim.location", "", "Location", "Geo Location"),
+            DataMapping("extras.tags", "", "Tag", "Device Group"),
+            DataMapping("dcim.device", "", "Device", "Device"),
+        )
 
     def load_source_adapter(self):
         """Load data from Nautobot into DiffSync models."""
@@ -202,14 +241,34 @@ def load_target_adapter(self):
         self.target_adapter = librenms.LibrenmsAdapter(job=self, sync=self.sync, librenms_api=librenms_api)
         self.target_adapter.load()
 
-    def run(self, dryrun, memory_profiling, debug, librenms_server, *args, **kwargs):  # pylint: disable=arguments-differ
+    def run(
+        self,
+        dryrun,
+        memory_profiling,
+        debug,
+        librenms_server,
+        force_add,
+        ping_fallback,
+        sync_locations,
+        location_type,
+        tenant,
+        *args,
+        **kwargs,
+    ):  # pylint: disable=arguments-differ
         """Perform data synchronization."""
         self.librenms_server = librenms_server
+        self.force_add = force_add
+        self.ping_fallback = ping_fallback
+        self.sync_locations = sync_locations
+        self.location_type = location_type
+        self.hostname_field == "env_var"
+        self.load_type == "api"
+        self.tenant = tenant
         self.debug = debug
         self.dryrun = dryrun
         self.memory_profiling = memory_profiling
         super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs)
 
 
-jobs = [LibrenmsDataSource]
+jobs = [LibrenmsDataSource, LibrenmsDataTarget]
 register_jobs(*jobs)