Merge pull request #35 from netboxlabs/develop

🚚 release

Makefile

@@ -14,10 +14,10 @@ docker-compose-netbox-plugin-down:
 
 .PHONY: docker-compose-netbox-plugin-test
 docker-compose-netbox-plugin-test:
-	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml run -u root --rm netbox ./manage.py test --keepdb netbox_diode_plugin
+	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run -u root --rm netbox ./manage.py test --keepdb netbox_diode_plugin
 	@$(MAKE) docker-compose-netbox-plugin-down
 
 .PHONY: docker-compose-netbox-plugin-test-cover
 docker-compose-netbox-plugin-test-cover:
-	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml run --rm -u root -e COVERAGE_FILE=/opt/netbox/netbox/coverage/.coverage netbox sh -c "coverage run --source=netbox_diode_plugin ./manage.py test --keepdb netbox_diode_plugin && coverage xml -o /opt/netbox/netbox/coverage/report.xml && coverage report -m | tee /opt/netbox/netbox/coverage/report.txt"
+	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run --rm -u root -e COVERAGE_FILE=/opt/netbox/netbox/coverage/.coverage netbox sh -c "coverage run --source=netbox_diode_plugin --omit=*_pb2*,*/migrations/* ./manage.py test --keepdb netbox_diode_plugin && coverage xml -o /opt/netbox/netbox/coverage/report.xml && coverage report -m | tee /opt/netbox/netbox/coverage/report.txt"
 	@$(MAKE) docker-compose-netbox-plugin-down

README.md

@@ -40,6 +40,30 @@ PLUGINS = [
 ]
 ```
 
+Also in your `configuration.py` file, in order to customise the plugin settings, add `netbox_diode_plugin`to the
+`PLUGINS_CONFIG` dictionary, e.g.:
+
+```python
+PLUGINS_CONFIG = {
+    "netbox_diode_plugin": {
+        # Diode gRPC target for communication with Diode server
+        "diode_target_override": "grpc://localhost:8080/diode",
+
+        # User allowed for Diode to NetBox communication
+        "diode_to_netbox_username": "diode-to-netbox",
+
+        # User allowed for NetBox to Diode communication
+        "netbox_to_diode_username": "netbox-to-diode",
+
+        # # User allowed for data ingestion
+        "diode_username": "diode-ingestion",
+    },
+}
+```
+
+Note: Once you customise usernames with PLUGINS_CONFIG during first installation, you should not change or remove them
+later on. Doing so will cause the plugin to stop working properly.
+
 Restart NetBox services to load the plugin:
 
 ```

docker/Dockerfile-diode-netbox-plugin

@@ -1,4 +1,4 @@
-FROM netboxcommunity/netbox:v4.1-beta1-2.9.1
+FROM netboxcommunity/netbox:v4.1-3.0.1
 
 COPY ./netbox/configuration/ /etc/netbox/config/
 RUN chmod 755 /etc/netbox/config/* && \

docker/docker-compose.test.yaml

@@ -0,0 +1,5 @@
+name: diode-netbox-plugin
+services:
+  netbox:
+    volumes:
+      - ./netbox/plugins_test.py:/etc/netbox/config/plugins.py:z,ro

docker/docker-compose.yaml

@@ -1,7 +1,7 @@
 name: diode-netbox-plugin
 services:
-  netbox: &netbox
-    image: netboxcommunity/netbox:v4.1-beta1-2.9.1-diode-netbox-plugin
+  netbox:
+    image: netboxcommunity/netbox:v4.1-3.0.1-diode-netbox-plugin
     build:
       context: .
       dockerfile: Dockerfile-diode-netbox-plugin

docker/netbox/configuration/plugins.py

@@ -8,6 +8,16 @@
 
 # PLUGINS_CONFIG = {
 #     "netbox_diode_plugin": {
+#         # Diode gRPC target for communication with Diode server
+#         "diode_target_override": "grpc://localhost:8080/diode",
 #
-#     }
+#         # User allowed for Diode to NetBox communication
+#         "diode_to_netbox_username": "diode-to-netbox",
+#
+#         # User allowed for NetBox to Diode communication
+#         "netbox_to_diode_username": "netbox-to-diode",
+#
+#         # User allowed for data ingestion
+#         "diode_username": "diode-ingestion",
+#     },
 # }

docker/netbox/launch-netbox.sh

@@ -46,10 +46,10 @@ load_configuration() {
 
 reload_netbox() {
   if [ "$RELOAD_NETBOX_ON_DIODE_PLUGIN_CHANGE" == "true" ]; then
-    netbox_diode_plugin_md5=$(find /opt/netbox/netbox/netbox_diode_plugin -type f -name "*.py" -exec md5sum {} + | md5sum | awk '{print $1}')
+    netbox_diode_plugin_md5=$(find /opt/netbox/netbox/netbox_diode_plugin -type f \( -name "*.py" -o -name "*.html" \) -exec md5sum {} + | md5sum | awk '{print $1}')
 
     while true; do
-      new_md5=$(find /opt/netbox/netbox/netbox_diode_plugin -type f -name "*.py" -exec md5sum {} + | md5sum | awk '{print $1}')
+      new_md5=$(find /opt/netbox/netbox/netbox_diode_plugin -type f \( -name "*.py" -o -name "*.html" \) -exec md5sum {} + | md5sum | awk '{print $1}')
       if [ "$netbox_diode_plugin_md5" != "$new_md5" ]; then
         echo "🔄 Reloading NetBox"
         curl --silent --output /dev/null --unix-socket /opt/unit/unit.sock -X GET http://localhost/control/applications/netbox/restart

docker/netbox/plugins_test.py

@@ -0,0 +1,13 @@
+# Add your plugins and plugin settings here.
+# Of course uncomment this file out.
+
+# To learn how to build images with your required plugins
+# See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins
+
+PLUGINS = ["netbox_diode_plugin"]
+
+PLUGINS_CONFIG = {
+    "netbox_diode_plugin": {
+        "enable_ingestion_logs": True,
+    }
+}

docker/requirements-diode-netbox-plugin.txt

@@ -1,3 +1,6 @@
-pytest==8.0.2
-grpcio==1.62.1
+Brotli==1.1.0
+certifi==2024.7.4
 coverage==7.6.0
+grpcio==1.62.1
+protobuf==5.28.1
+pytest==8.0.2

netbox_diode_plugin/__init__.py

@@ -16,6 +16,19 @@ class NetBoxDiodePluginConfig(PluginConfig):
     version = version_semver()
     base_url = "diode"
     min_version = "3.7.2"
+    default_settings = {
+        # Default Diode gRPC target for communication with Diode server
+        "diode_target": "grpc://localhost:8080/diode",
+
+        # User allowed for Diode to NetBox communication
+        "diode_to_netbox_username": "diode-to-netbox",
+
+        # User allowed for NetBox to Diode communication
+        "netbox_to_diode_username": "netbox-to-diode",
+
+        # User allowed for data ingestion
+        "diode_username": "diode-ingestion",
+    }
 
 
 config = NetBoxDiodePluginConfig

netbox_diode_plugin/api/serializers.py

@@ -14,15 +14,17 @@
     SiteSerializer,
 )
 from django.conf import settings
+from netbox.api.serializers import NetBoxModelSerializer
 from packaging import version
 
+from netbox_diode_plugin.models import Setting
+
 if version.parse(version.parse(settings.VERSION).base_version) >= version.parse("4.1"):
     from core.models import ObjectChange
 else:
     from extras.models import ObjectChange
 from ipam.api.serializers import IPAddressSerializer, PrefixSerializer
 from rest_framework import serializers
-from rest_framework.utils.serializer_helpers import ReturnDict
 from utilities.api import get_serializer_for_model
 from virtualization.api.serializers import (
     ClusterGroupSerializer,
@@ -126,6 +128,22 @@ class ApplyChangeSetRequestSerializer(serializers.Serializer):
     )
 
 
+class SettingSerializer(NetBoxModelSerializer):
+    """Setting Serializer."""
+
+    class Meta:
+        """Meta class."""
+
+        model = Setting
+        fields = (
+            "id",
+            "diode_target",
+            "custom_fields",
+            "created",
+            "last_updated",
+        )
+
+
 class DiodeIPAddressSerializer(IPAddressSerializer):
     """Diode IP Address Serializer."""
 

netbox_diode_plugin/forms.py

@@ -0,0 +1,40 @@
+# !/usr/bin/env python
+# Copyright 2024 NetBox Labs Inc
+"""Diode NetBox Plugin - Forms."""
+from netbox.forms import NetBoxModelForm
+from netbox.plugins import get_plugin_config
+from utilities.forms.rendering import FieldSet
+
+from netbox_diode_plugin.models import Setting
+
+__all__ = ("SettingsForm",)
+
+
+class SettingsForm(NetBoxModelForm):
+    """Settings form."""
+
+    fieldsets = (
+        FieldSet(
+            "diode_target",
+        ),
+    )
+
+    class Meta:
+        """Meta class."""
+
+        model = Setting
+        fields = ("diode_target",)
+
+    def __init__(self, *args, **kwargs):
+        """Initialize the form."""
+        super().__init__(*args, **kwargs)
+
+        diode_target_override = get_plugin_config(
+            "netbox_diode_plugin", "diode_target_override"
+        )
+
+        if diode_target_override:
+            self.fields["diode_target"].disabled = True
+            self.fields["diode_target"].help_text = (
+                "This field is not allowed to be modified."
+            )

netbox_diode_plugin/migrations/0001_initial.py

@@ -5,14 +5,29 @@
 import os
 
 from django.apps import apps as django_apps
-from django.conf import settings
+from django.conf import settings as netbox_settings
 from django.contrib.contenttypes.management import create_contenttypes
 from django.db import migrations, models
 from users.models import Token as NetBoxToken
 
+from netbox_diode_plugin.plugin_config import get_diode_usernames
 
-def _create_user_with_token(apps, username, group, is_superuser: bool = False):
-    User = apps.get_model(settings.AUTH_USER_MODEL)
+
+# Read secret from file
+def _read_secret(secret_name, default=None):
+    try:
+        f = open("/run/secrets/" + secret_name, encoding="utf-8")
+    except OSError:
+        return default
+    else:
+        with f:
+            return f.readline().strip()
+
+
+def _create_user_with_token(
+    apps, user_category, username, group, is_superuser: bool = False
+):
+    User = apps.get_model(netbox_settings.AUTH_USER_MODEL)
     """Create a user with the given username and API key if it does not exist."""
     try:
         user = User.objects.get(username=username)
@@ -27,7 +42,8 @@ def _create_user_with_token(apps, username, group, is_superuser: bool = False):
     Token = apps.get_model("users", "Token")
 
     if not Token.objects.filter(user=user).exists():
-        api_key = os.getenv(f"{username}_API_KEY")
+        key = f"{user_category.upper()}_API_KEY"
+        api_key = _read_secret(key.lower(), os.getenv(key))
         if api_key is None:
             api_key = NetBoxToken.generate_key()
         Token.objects.create(user=user, key=api_key)
@@ -37,18 +53,18 @@ def _create_user_with_token(apps, username, group, is_superuser: bool = False):
 
 def configure_plugin(apps, schema_editor):
     """Configure the plugin."""
-    diode_to_netbox_username = "DIODE_TO_NETBOX"
-    netbox_to_diode_username = "NETBOX_TO_DIODE"
-    diode_username = "DIODE"
-
     Group = apps.get_model("users", "Group")
     group, _ = Group.objects.get_or_create(name="diode")
 
-    diode_to_netbox_user = _create_user_with_token(
-        apps, diode_to_netbox_username, group
-    )
-    _ = _create_user_with_token(apps, netbox_to_diode_username, group, True)
-    _ = _create_user_with_token(apps, diode_username, group)
+    diode_to_netbox_user_id = None
+
+    for user_category, username in get_diode_usernames().items():
+        is_superuser = user_category in ("netbox_to_diode",)
+        user = _create_user_with_token(
+            apps, user_category, username, group, is_superuser
+        )
+        if user_category == "diode_to_netbox":
+            diode_to_netbox_user_id = user.id
 
     app_config = django_apps.get_app_config("netbox_diode_plugin")
 
@@ -66,8 +82,7 @@ def configure_plugin(apps, schema_editor):
         actions=["add", "view"],
     )
 
-    permission.groups.set([group.id])
-    permission.users.set([diode_to_netbox_user.id])
+    permission.users.set([diode_to_netbox_user_id])
     permission.object_types.set([diode_plugin_object_type.id])
 
 

netbox_diode_plugin/migrations/0002_setting.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Copyright 2024 NetBox Labs Inc
+"""Diode Netbox Plugin - Database migrations."""
+
+import utilities.json
+from django.db import migrations, models
+from netbox.plugins import get_plugin_config
+
+
+def create_settings_entity(apps, schema_editor):
+    """Create a Setting entity."""
+    Setting = apps.get_model("netbox_diode_plugin", "Setting")
+
+    default_diode_target = get_plugin_config("netbox_diode_plugin", "diode_target")
+    diode_target = get_plugin_config(
+        "netbox_diode_plugin", "diode_target_override", default_diode_target
+    )
+
+    Setting.objects.create(diode_target=diode_target)
+
+
+class Migration(migrations.Migration):
+    """0002_setting migration."""
+
+    dependencies = [
+        ("netbox_diode_plugin", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Setting",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True, null=True)),
+                ("last_updated", models.DateTimeField(auto_now=True, null=True)),
+                (
+                    "custom_field_data",
+                    models.JSONField(
+                        blank=True,
+                        default=dict,
+                        encoder=utilities.json.CustomFieldJSONEncoder,
+                    ),
+                ),
+                ("diode_target", models.CharField(max_length=255)),
+            ],
+            options={
+                "verbose_name": "Setting",
+                "verbose_name_plural": "Diode Settings",
+            },
+        ),
+        migrations.RunPython(
+            code=create_settings_entity, reverse_code=migrations.RunPython.noop
+        ),
+    ]

netbox_diode_plugin/migrations/0003_clear_permissions.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# Copyright 2024 NetBox Labs Inc
+"""Diode Netbox Plugin - Database migrations."""
+
+from django.db import migrations
+
+
+def clear_diode_group_permissions(apps, schema_editor):
+    """Clear Diode group permissions."""
+    ObjectPermission = apps.get_model("users", "ObjectPermission")
+    permission = ObjectPermission.objects.get(name="Diode")
+    permission.groups.clear()
+
+
+class Migration(migrations.Migration):
+    """0003_clear_permissions migration."""
+
+    dependencies = [
+        ("netbox_diode_plugin", "0001_initial"),
+        ("netbox_diode_plugin", "0002_setting"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            code=clear_diode_group_permissions, reverse_code=migrations.RunPython.noop
+        ),
+    ]