Fix Main (#152)

* chore: 🚚 Update test file name to match filepath of file being tested

* style: 🚨 Fix formatting for black

* build: Update to latest syntax for compose

* build: Use Postgres 13 for Apple ARM support

* fix: 🐛 Add check for existence of Relationship in source list before returning.

Also refactored slightly to have it return a sorted list of VLANs based on VLAN ID

* test: ✅ Add tests validating Nautobot utility methods

* build: Update lockfile for latest dependencies

* build: Set upper bound on nautobot-ssot to prevent conflict.

* build: Update Dockerfile to latest pattern pinning poetry.

* ci: Update CI to latest pattern and lock Nautobot version.

* build: Remove extras all

* ci: Remove py3.7 from unit tests

* ci: Remove mySQL tests, simplify unittest matrix

.github/workflows/ci.yml

@@ -25,8 +25,6 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
-        with:
-          python-version: "3.10"
       - name: "Linting: black"
         run: "poetry run invoke black"
   bandit:
@@ -38,8 +36,6 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
-        with:
-          python-version: "3.10"
       - name: "Linting: bandit"
         run: "poetry run invoke bandit"
   pydocstyle:
@@ -51,8 +47,6 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
-        with:
-          python-version: "3.10"
       - name: "Linting: pydocstyle"
         run: "poetry run invoke pydocstyle"
   flake8:
@@ -64,8 +58,6 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
-        with:
-          python-version: "3.10"
       - name: "Linting: flake8"
         run: "poetry run invoke flake8"
   yamllint:
@@ -77,8 +69,6 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
-        with:
-          python-version: "3.10"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
   pylint:
@@ -93,7 +83,7 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.8"]
-        nautobot-version: ["1.4.0"]
+        nautobot-version: ["1.6.0"]
     env:
       INVOKE_NAUTOBOT_SSOT_INFOBLOX_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_SSOT_INFOBLOX_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
@@ -102,8 +92,23 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
+      - name: "Set up Docker Buildx"
+        id: "buildx"
+        uses: "docker/setup-buildx-action@v1"
+      - name: "Build"
+        uses: "docker/build-push-action@v2"
         with:
-          python-version: "3.10"
+          builder: "${{ steps.buildx.outputs.name }}"
+          context: "./"
+          push: false
+          load: true
+          tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          file: "./development/Dockerfile"
+          cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          build-args: |
+            NAUTOBOT_VER=${{ matrix.nautobot-version }}
+            PYTHON_VER=${{ matrix.python-version }}
       - name: "Copy credentials"
         run: "cp development/creds.example.env development/creds.env"
       - name: "Linting: pylint"
@@ -114,8 +119,9 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        nautobot-version: ["1.4.2", "latest"]
+        python-version: ["3.8", "3.9", "3.10"]
+        db-backend: ["postgresql"]
+        nautobot-version: ["1.5.0"]
     runs-on: "ubuntu-20.04"
     env:
       INVOKE_NAUTOBOT_SSOT_INFOBLOX_PYTHON_VER: "${{ matrix.python-version }}"
@@ -125,12 +131,25 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
+      - name: "Set up Docker Buildx"
+        id: "buildx"
+        uses: "docker/setup-buildx-action@v1"
+      - name: "Build"
+        uses: "docker/build-push-action@v2"
         with:
-          python-version: "${{ matrix.python-version }}"
+          builder: "${{ steps.buildx.outputs.name }}"
+          context: "./"
+          push: false
+          load: true
+          tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          file: "./development/Dockerfile"
+          cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          build-args: |
+            NAUTOBOT_VER=${{ matrix.nautobot-version }}
+            PYTHON_VER=${{ matrix.python-version }}
       - name: "Copy credentials"
         run: "cp development/creds.example.env development/creds.env"
-      - name: "Build Container"
-        run: "poetry run invoke build"
       - name: "Run Tests"
         run: "poetry run invoke unittest"
   publish_gh:
@@ -188,38 +207,38 @@ jobs:
         with:
           user: "__token__"
           password: "${{ secrets.PYPI_API_TOKEN }}"
-  slack-notify:
-    needs:
-      - "publish_gh"
-      - "publish_pypi"
-    runs-on: "ubuntu-20.04"
-    env:
-      SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
-      SLACK_MESSAGE: >-
-        *NOTIFICATION: NEW-RELEASE-PUBLISHED*\n
-        Repository: <${{ github.server_url }}/${{ github.repository }}|${{ github.repository }}>\n
-        Release: <${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>\n
-        Published by: <${{ github.server_url }}/${{ github.actor }}|${{ github.actor }}>
-    steps:
-      - name: "Send a notification to Slack"
-        # ENVs cannot be used directly in job.if. This is a workaround to check
-        # if SLACK_WEBHOOK_URL is present.
-        if: "env.SLACK_WEBHOOK_URL != ''"
-        uses: "slackapi/[email protected]"
-        with:
-          payload: |
-            {
-              "text": "${{ env.SLACK_MESSAGE }}",
-              "blocks": [
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "${{ env.SLACK_MESSAGE }}"
-                  }
-                }
-              ]
-            }
-        env:
-          SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
-          SLACK_WEBHOOK_TYPE: "INCOMING_WEBHOOK"
+  # slack-notify:
+  #   needs:
+  #     - "publish_gh"
+  #     - "publish_pypi"
+  #   runs-on: "ubuntu-20.04"
+  #   env:
+  #     SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
+  #     SLACK_MESSAGE: >-
+  #       *NOTIFICATION: NEW-RELEASE-PUBLISHED*\n
+  #       Repository: <${{ github.server_url }}/${{ github.repository }}|${{ github.repository }}>\n
+  #       Release: <${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>\n
+  #       Published by: <${{ github.server_url }}/${{ github.actor }}|${{ github.actor }}>
+  #   steps:
+  #     - name: "Send a notification to Slack"
+  #       # ENVs cannot be used directly in job.if. This is a workaround to check
+  #       # if SLACK_WEBHOOK_URL is present.
+  #       if: "env.SLACK_WEBHOOK_URL != ''"
+  #       uses: "slackapi/[email protected]"
+  #       with:
+  #         payload: |
+  #           {
+  #             "text": "${{ env.SLACK_MESSAGE }}",
+  #             "blocks": [
+  #               {
+  #                 "type": "section",
+  #                 "text": {
+  #                   "type": "mrkdwn",
+  #                   "text": "${{ env.SLACK_MESSAGE }}"
+  #                 }
+  #               }
+  #             ]
+  #           }
+  #       env:
+  #         SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
+  #         SLACK_WEBHOOK_TYPE: "INCOMING_WEBHOOK"

development/Dockerfile

@@ -1,18 +1,18 @@
 # -------------------------------------------------------------------------------------
 # Nautobot App Developement Dockerfile Template
-# Version: 1.0.0
+# Version: 1.1.0
 #
 # Apps that need to add additional steps or packages can do in the section below.
 # -------------------------------------------------------------------------------------
 # !!! USE CAUTION WHEN MODIFYING LINES BELOW
 
-# Accepts a desired Nautobot version as build argument, default to 1.4.0
+# Accepts a desired Nautobot version as build argument, default to 1.4
 ARG NAUTOBOT_VER="1.4"
 
 # Accepts a desired Python version as build argument, default to 3.8
 ARG PYTHON_VER="3.8"
 
-# Retreive published development image of Nautobot base which should include most CI dependencies
+# Retrieve published development image of Nautobot base which should include most CI dependencies
 FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER}
 
 # Runtime argument and environment setup
@@ -26,10 +26,9 @@ ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT}
 # and CI and local development may have a newer version of Poetry
 # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary
 # We also don't need virtual environments in container
-RUN curl -sSL https://install.python-poetry.org -o /tmp/install-poetry.py && \
-    python /tmp/install-poetry.py && \
-    rm -f /tmp/install-poetry.py && \
-    poetry config virtualenvs.create false
+ENV POETRY_VERSION=1.5.1
+RUN curl -sSL https://install.python-poetry.org | python3 - && \
+  poetry config virtualenvs.create false
 
 # !!! USE CAUTION WHEN MODIFYING LINES ABOVE
 # -------------------------------------------------------------------------------------
@@ -39,10 +38,6 @@ RUN curl -sSL https://install.python-poetry.org -o /tmp/install-poetry.py && \
 # -------------------------------------------------------------------------------------
 # --> Start safe to modify section
 
-# Uncomment the line below if you are apt-installing any package.
-# RUN apt update
-# RUN apt install libldap2-dev
-
 # --> Stop safe to modify section
 # -------------------------------------------------------------------------------------
 # Install Nautobot App
@@ -64,16 +59,18 @@ RUN pip show nautobot | grep "^Version: " | sed -e 's/Version: /nautobot==/' > c
 # We can't use the entire freeze as it takes forever to resolve with rigidly fixed non-direct dependencies,
 #   especially those that are only direct to Nautobot but the container included versions slightly mismatch
 RUN poetry export -f requirements.txt --without-hashes --output poetry_freeze_base.txt
-RUN poetry export -f requirements.txt --dev --without-hashes --output poetry_freeze_all.txt
+RUN poetry export -f requirements.txt --without-hashes --with dev --output poetry_freeze_all.txt
 RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_dev.txt
 
 # Install all local project as editable, constrained on Nautobot version, to get any additional
 #   direct dependencies of the app
-RUN pip install -c constraints.txt -e .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -c constraints.txt -e .[all]
 
 # Install any dev dependencies frozen from Poetry
 # Can be improved in Poetry 1.2 which allows `poetry install --only dev`
-RUN pip install -c constraints.txt -r poetry_freeze_dev.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -c constraints.txt -r poetry_freeze_dev.txt
 
 COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py
 # !!! USE CAUTION WHEN MODIFYING LINES ABOVE

development/docker-compose.base.yml

@@ -19,8 +19,9 @@ services:
     depends_on:
       - "postgres"
       - "redis"
-    <<: *nautobot-build
-    <<: *nautobot-base
+    <<:
+      - *nautobot-build
+      - *nautobot-base
   worker:
     entrypoint:
       - "sh"

development/docker-compose.requirements.yml

@@ -2,7 +2,7 @@
 version: "3.8"
 services:
   postgres:
-    image: "postgres:14-alpine"
+    image: "postgres:13-alpine"
     env_file:
       - "development.env"
       - "creds.env"

nautobot_ssot_infoblox/tests/test_utils_nautobot.py

@@ -0,0 +1,57 @@
+"""Test utility methods for Nautobot."""
+from django.contrib.contenttypes.models import ContentType
+from nautobot.extras.models import Relationship, RelationshipAssociation, Status
+from nautobot.ipam.models import Prefix, VLAN, VLANGroup
+from nautobot.utilities.testing import TransactionTestCase
+from nautobot_ssot_infoblox.utils.nautobot import build_vlan_map_from_relations, get_prefix_vlans
+
+
+class TestNautobotUtils(TransactionTestCase):
+    """Test Nautobot Utility methods."""
+
+    def setUp(self):
+        """Configure common objects for tests."""
+        super().setUp()
+        self.status_active = Status.objects.get(name="Active")
+        self.test_pf = Prefix.objects.get_or_create(prefix="192.168.1.0/24")[0]
+        self.vlan_group = VLANGroup.objects.create(name="Test")
+        self.test_vlan1 = VLAN.objects.create(name="Test1", vid=1, status=self.status_active, group=self.vlan_group)
+        self.test_vlan1.validated_save()
+        self.test_vlan2 = VLAN.objects.create(name="Test2", vid=2, status=self.status_active, group=self.vlan_group)
+        self.test_vlan2.validated_save()
+
+    def test_build_vlan_map_from_relations(self):
+        """Validate functionality of the build_vlan_map_from_relations() function."""
+        test_list = [self.test_vlan1, self.test_vlan2]
+        actual = build_vlan_map_from_relations(vlans=test_list)
+        expected = {1: {"vid": 1, "name": "Test1", "group": "Test"}, 2: {"vid": 2, "name": "Test2", "group": "Test"}}
+        self.assertEqual(actual, expected)
+
+    def test_get_prefix_vlans_success(self):
+        """Validate functionality of the get_prefix_vlans() function success."""
+        pf_vlan_rel = Relationship.objects.get(slug="prefix_to_vlan")
+        rel_assoc1 = RelationshipAssociation.objects.create(
+            relationship_id=pf_vlan_rel.id,
+            source_type=ContentType.objects.get_for_model(Prefix),
+            source_id=self.test_pf.id,
+            destination_type=ContentType.objects.get_for_model(VLAN),
+            destination_id=self.test_vlan1.id,
+        )
+        rel_assoc1.validated_save()
+        rel_assoc2 = RelationshipAssociation.objects.create(
+            relationship_id=pf_vlan_rel.id,
+            source_type=ContentType.objects.get_for_model(Prefix),
+            source_id=self.test_pf.id,
+            destination_type=ContentType.objects.get_for_model(VLAN),
+            destination_id=self.test_vlan2.id,
+        )
+        rel_assoc2.validated_save()
+        expected = [self.test_vlan1, self.test_vlan2]
+        actual = get_prefix_vlans(self.test_pf)
+        self.assertEqual(actual, expected)
+
+    def test_get_prefix_vlans_failure(self):
+        """Validate functionality of the get_prefix_vlans() function failure where Prefix has no RelationshipAssocations to VLANs."""
+        expected = []
+        actual = get_prefix_vlans(self.test_pf)
+        self.assertEqual(actual, expected)

nautobot_ssot_infoblox/utils/nautobot.py

@@ -26,6 +26,10 @@ def get_prefix_vlans(prefix: Prefix) -> list:
     Returns:
         list: List of VLAN objects with RelationshipAssociation to passed Prefix.
     """
+    vlan_list = []
     pf_relations = prefix.get_relationships()
     pf_vlan_relationship = Relationship.objects.get(name="Prefix -> VLAN")
-    return [x.destination for x in pf_relations["source"][pf_vlan_relationship]]
+    if pf_vlan_relationship in pf_relations["source"]:
+        vlan_list = [x.destination for x in pf_relations["source"][pf_vlan_relationship]]
+        vlan_list.sort(key=lambda x: x.vid)
+    return vlan_list