wip

pyproject.toml

@@ -15,6 +15,7 @@ django-extensions = "^3.2.3"
 celery = {extras = ["redis"], version = "^5.3.6"}
 chardet = "^5.2.0"
 django-import-export = "^3.3.6"
+tablib = "^3.5.0"
 
 
 [tool.poetry.group.dev.dependencies]

sensor/api/viewsets.py

@@ -10,4 +10,4 @@ class FileViewSet(viewsets.ModelViewSet):
 
     def perform_create(self, serializer):
         instance = serializer.save()
-        instance.import_to_db()
+        instance.parse_and_import_to_db()

sensor/models.py

@@ -1,3 +1,5 @@
+
+import json
 import textwrap
 
 from datetime import datetime
@@ -122,7 +124,31 @@ def clean(self):
         )
         self.file.seek(0)
 
-    def import_to_db(self):
+    def import_directly_to_db(self):
+
+        # NOTE: assume uploaded file is JSON
+        with self.file.open(mode="rb") as f:
+
+            reading_objs = (
+                Reading(
+                    file=self,
+                    timestamp=r["timestamp"],
+                    sensor_name=r["sensor_name"],
+                    reading=r["reading"]
+                )
+                for r in json.load(f)
+            )
+
+            batch_size = 1_000
+
+            with transaction.atomic():
+                while True:
+                    batch = list(islice(reading_objs, batch_size))
+                    if not batch:
+                        break
+                    Reading.objects.bulk_create(batch, batch_size)
+
+    def parse_and_import_to_db(self):
 
         with self.file.open(mode="rb") as f:
 
@@ -163,6 +189,10 @@ def import_to_db(self):
                 self.parse_error = None
                 self.save()
 
+    def import_to_db(self) -> None:
+
+        self.import_directly()
+
 
 class Reading(models.Model):
 

sensor/tasks.py

@@ -4,6 +4,6 @@
 
 
 @shared_task
-def import_to_db(file_id):
+def parse_and_import_to_db(file_id):
     file_obj = File.objects.get(id=file_id)
-    file_obj.import_to_db()
+    file_obj.parse_and_import_to_db()

sensor/views.py

@@ -23,7 +23,7 @@ def upload_file(request):
         form = FileForm(request.POST, request.FILES)
         if form.is_valid():
             form.save()
-            form.instance.import_to_db()
+            form.instance.parse_and_import_to_db()
             return HttpResponse("File upload was successful")
         else:
             return HttpResponse(f"File type creation failed: {form.errors}")

tests/sensor/__snapshots__/test_models.ambr

@@ -1,4 +1,7 @@
 # serializer version: 1
-# name: test_import_to_db[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0]
+# name: test_import_directly_to_db[lines0]
+  <QuerySet [<Reading: M(m/s) @ 2015-12-22 00:00:00 = 20.54>, <Reading: D(deg) @ 2015-12-22 00:00:00 = 211.0>]>
+# ---
+# name: test_parse_and_import_to_db[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0]
   <QuerySet [<Reading: M(m/s) @ 2015-12-22 00:00:00 = 20.54>, <Reading: D(deg) @ 2015-12-22 00:00:00 = 211.0>, <Reading: SD(m/s) @ 2015-12-22 00:00:00 = 1.22>, <Reading: DSD(deg) @ 2015-12-22 00:00:00 = 0.3>, <Reading: Gust3s(m/s) @ 2015-12-22 00:00:00 = 21.00>, <Reading: T(C) @ 2015-12-22 00:00:00 = 11.9>, <Reading: PRE(hPa) @ 2015-12-22 00:00:00 = 992.8>, <Reading: RiNumber @ 2015-12-22 00:00:00 = 0.15>, <Reading: VertM(m/s) @ 2015-12-22 00:00:00 = 0.18>, <Reading:  @ 2015-12-22 00:00:00 = >, <Reading: M(m/s) @ 2015-12-22 00:10:00 = 21.02>, <Reading: D(deg) @ 2015-12-22 00:10:00 = 212.2>, <Reading: SD(m/s) @ 2015-12-22 00:10:00 = 2.55>, <Reading: DSD(deg) @ 2015-12-22 00:10:00 = 0.6>, <Reading: Gust3s(m/s) @ 2015-12-22 00:10:00 = 21.35>, <Reading: T(C) @ 2015-12-22 00:10:00 = 11.8>, <Reading: PRE(hPa) @ 2015-12-22 00:10:00 = 992.7>, <Reading: RiNumber @ 2015-12-22 00:10:00 = 0.29>, <Reading: VertM(m/s) @ 2015-12-22 00:10:00 = -0.09>]>
 # ---

tests/sensor/api/test_viewsets.py

@@ -106,7 +106,7 @@ def test_cannot_upload_a_missing_file_type(
         assert response.status_code == HTTPStatus.BAD_REQUEST
         assert response.content == snapshot
 
-    def test_import_to_db_is_called(
+    def test_parse_and_import_to_db_is_called(
         self,
         client,
         lines,
@@ -129,6 +129,6 @@ def test_import_to_db_is_called(
         )
         url = reverse("sensor:upload-file")
 
-        with patch("sensor.models.File.import_to_db") as importer:
+        with patch("sensor.models.File.parse_and_import_to_db") as importer:
             client.post(url, {"file": file, "type": file_type.id})
             assert importer.called

tests/sensor/test_models.py

@@ -1,13 +1,49 @@
+import datetime
+import json
+
 from django.core.files.base import ContentFile
 import pytest
 
 from sensor.models import File
 from sensor.models import FileType
 from sensor.models import Reading
-
 from tests.globals import SOURCES
 
 
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "lines",
+    [
+        [
+            {
+                'reading': '20.54',
+                'sensor_name': 'M(m/s)',
+                'timestamp': str(datetime.datetime(2015, 12, 22, 0, 0)),
+            },
+            {
+                'reading': '211.0',
+                'sensor_name': 'D(deg)',
+                'timestamp': str(datetime.datetime(2015, 12, 22, 0, 0)),
+            },
+        ]
+    ]
+)
+def test_import_directly_to_db(
+    lines,
+    snapshot,
+) -> None:
+
+    file = ContentFile(json.dumps(lines), name="sensor-readings.txt")
+    file_type_obj = FileType.objects.create(name="file-type")
+    file_obj = File(file=file, type=file_type_obj)
+    file_obj.save()
+
+    file_obj.import_directly_to_db()
+
+    output = Reading.objects.all()
+    assert output == snapshot
+
+
 @pytest.mark.django_db
 @pytest.mark.parametrize(
     "lines,encoding,delimiter,datetime_fieldnames,datetime_formats,na_values",
@@ -23,7 +59,7 @@
         for source in SOURCES
     ]
 )
-def test_import_to_db(
+def test_parse_and_import_to_db(
     lines,
     encoding,
     delimiter,
@@ -34,6 +70,7 @@ def test_import_to_db(
 ) -> None:
 
     file_type_obj = FileType.objects.create(
+        name="file-type",
         delimiter=delimiter,
         datetime_fieldnames=datetime_fieldnames,
         datetime_formats=datetime_formats,
@@ -44,7 +81,7 @@ def test_import_to_db(
     file_obj = File(file=file, type=file_type_obj)
     file_obj.save()
 
-    file_obj.import_to_db()
+    file_obj.parse_and_import_to_db()
 
     output = Reading.objects.all()
     assert output == snapshot

tests/sensor/test_tasks.py

@@ -5,17 +5,17 @@
 
 from sensor.models import File
 from sensor.models import FileType
-from sensor.tasks import import_to_db
+from sensor.tasks import parse_and_import_to_db
 
 
 @pytest.mark.django_db
-def test_import_to_db_is_called():
+def test_parse_and_import_to_db_is_called():
 
-    with patch("sensor.models.File.import_to_db") as importer:
+    with patch("sensor.models.File.parse_and_import_to_db") as importer:
         file_obj = File.objects.create(
             file=ContentFile(b"", name="sensor-readings.txt"),
             type=FileType.objects.create(name="type")
         )
-        import_to_db(file_id=file_obj.id)
+        parse_and_import_to_db(file_id=file_obj.id)
 
-    assert importer.called
+    assert importer.called, "parse_and_import_to_db was not called!"

tests/sensor/test_views.py

@@ -81,7 +81,7 @@ def test_cannot_upload_an_invalid_file(
         assert response.status_code == HTTPStatus.OK
         assert response.content == snapshot
 
-    def test_import_to_db_is_called(
+    def test_parse_and_import_to_db_is_called(
         self,
         client,
         lines,
@@ -104,6 +104,6 @@ def test_import_to_db_is_called(
         )
         url = reverse("sensor:upload-file")
 
-        with patch("sensor.models.File.import_to_db") as importer:
+        with patch("sensor.models.File.parse_and_import_to_db") as importer:
             client.post(url,{"file": file, "type": file_type.id})
             assert importer.called