Faster node endpoint (#51)

* updated prefetch_related in NodesViewSet

* added django-debug-toolbar, it was very useful when trying to make the api faster

* added a prefetch to manifest api endpoint to make it faster

* added a condition in case node.phase is empty

* made more api endpoints faster

* only add debug_toolbar.urls when django in debug mode

* moving django-debug-toolbar to base requirements so collectstatic step works. left comment to fix later.

---------

Co-authored-by: Sean Shahkarami <[email protected]>

config/settings/dev.py

@@ -13,3 +13,10 @@
         "NAME": BASE_DIR / "db.sqlite3",
     }
 }
+
+#debug toolbar requirements
+INSTALLED_APPS.append("debug_toolbar")
+MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
+INTERNAL_IPS = [
+    "127.0.0.1",
+]

config/urls.py

@@ -13,6 +13,7 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.conf import settings
 from django.contrib import admin
 from django.urls import path, include
@@ -40,3 +41,8 @@
     ),
     path("downloads/", include("downloads.urls")),
 ]
+
+if settings.DEBUG:
+    urlpatterns += [
+        path("_debug/", include("debug_toolbar.urls")),
+    ]

manifests/serializers.py

@@ -38,7 +38,8 @@ def get_vsns(self, obj):
             nodes = [
                 node
                 for node in nodes
-                if node.phase.lower() in (p.lower() for p in phases)
+                if node.phase
+                and node.phase.lower() in (p.lower() for p in phases)
             ]
 
         vsns = sorted(set([node.vsn for node in nodes]))

manifests/views.py

@@ -44,6 +44,7 @@ def get_queryset(self):
                 "compute_set__computesensor_set__hardware__capabilities",
                 "compute_set__computesensor_set__labels",
                 "resource_set__hardware__capabilities",
+                "lorawanconnections__lorawan_device__hardware__capabilities",
                 "tags",
             )
             .order_by("vsn")
@@ -57,7 +58,14 @@ def get_queryset(self):
 
 
 class ComputeViewSet(ReadOnlyModelViewSet):
-    queryset = Compute.objects.all().order_by("node__vsn")
+    queryset = (
+        Compute.objects.all()
+        .prefetch_related(
+            "hardware",
+            "node",
+        )
+        .order_by("node__vsn")
+    )
     serializer_class = ComputeSerializer
     permission_classes = [IsAuthenticatedOrReadOnly]
 
@@ -66,20 +74,27 @@ class SensorHardwareViewSet(ReadOnlyModelViewSet):
     queryset = (
         SensorHardware.objects.all()
         .prefetch_related(
-            "nodesensor_set",
             "nodesensor_set__node",
-            "computesensor_set",
-            "computesensor_set__scope",
             "computesensor_set__scope__node",
-            "lorawandevice_set",
-            "lorawandevice_set__lorawanconnections",
+            "capabilities",
+            "lorawandevice_set__lorawanconnections__node",
         )
         .order_by("hardware")
     )
     serializer_class = SensorViewSerializer
     lookup_field = "hardware"
     permission_classes = [IsAuthenticatedOrReadOnly]
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.request.query_params.get("project"):
+            queryset = queryset.prefetch_related(
+                "lorawandevice_set__lorawanconnections__node__project",
+                "computesensor_set__scope__node__project",
+                "nodesensor_set__node__project",
+            )
+        return queryset
+
     def list(self, request, *args, **kwargs):
         res = super(SensorHardwareViewSet, self).list(request, *args, **kwargs)
 
@@ -91,7 +106,12 @@ def list(self, request, *args, **kwargs):
         return res
 
 class SensorHardwareViewSet_CRUD(NodeAuthMixin, ModelViewSet):
-    queryset = SensorHardware.objects.all()
+    queryset = (
+        SensorHardware.objects.all()
+        .prefetch_related(
+            "capabilities",
+        )
+    )
     serializer_class = SensorHardwareCRUDSerializer
     lookup_field = "hw_model" #TODO: this can cause a "multiple response returned" server error on GET as hw_model is not unique - FL 01/29/2024
     authentication_classes = (NodeAuthMixin.authentication_classes[0],UserTokenAuthentication)
@@ -100,7 +120,15 @@ class SensorHardwareViewSet_CRUD(NodeAuthMixin, ModelViewSet):
 class NodeBuildViewSet(ReadOnlyModelViewSet):
     queryset = (
         NodeBuild.objects.all()
-        .prefetch_related("top_camera", "bottom_camera", "left_camera", "right_camera")
+        .prefetch_related(
+            "top_camera", 
+            "bottom_camera", 
+            "left_camera", 
+            "right_camera",
+            "project",
+            "focus",
+            "partner",
+        )
         .order_by("vsn")
     )
     serializer_class = NodeBuildSerializer
@@ -109,13 +137,23 @@ class NodeBuildViewSet(ReadOnlyModelViewSet):
 
 
 class LorawanDeviceView(NodeAuthMixin, ModelViewSet):
-    queryset = LorawanDevice.objects.all()
+    queryset = (
+        LorawanDevice.objects.all()
+        .prefetch_related(
+            "labels"
+        )
+    )
     serializer_class = LorawanDeviceSerializer
     lookup_field = "deveui"
 
 
 class LorawanConnectionView(NodeOwnedObjectsMixin, ModelViewSet):
-    queryset = LorawanConnection.objects.all()
+    queryset = (
+        LorawanConnection.objects.all()
+        .prefetch_related(
+            "node",
+            "lorawan_device",
+        ))
     serializer_class = LorawanConnectionSerializer
     vsn_field = "node__vsn"
 
@@ -203,10 +241,11 @@ class NodesViewSet(ReadOnlyModelViewSet):
     queryset = (
         NodeData.objects.all()
         .prefetch_related(
-            "nodesensor_set",
-            "compute_set",
-            "lorawanconnections",
-            "modem"
+            "modem",
+            "lorawanconnections__lorawan_device__hardware__capabilities",
+            "compute_set__hardware__capabilities",
+            "nodesensor_set__hardware__capabilities",
+            "project",
         )
         .order_by("vsn")
     )

requirements/base.txt

@@ -16,3 +16,6 @@ django-filter==23.5
 git+https://github.com/waggle-sensor/django-address
 # setuptools needed for python 3.12+
 setuptools==69.2.0
+# NOTE(sean) I'm installing django-debug-toolbar in base to make the collectstatic step work
+# when building the image. We can revisit this later if we want to move it to just prod.
+django-debug-toolbar==4.3.0 # enables a debug toolbar, helps with debugging api endpoints