From dfd9b4df608afc9a5773a05eb265427ae8027596 Mon Sep 17 00:00:00 2001 From: Rowan Molony Date: Mon, 8 Jan 2024 15:32:55 +0000 Subject: [PATCH] wip --- sensor/api/serializers.py | 6 + sensor/api/viewsets.py | 3 - sensor/api_urls.py | 1 - sensor/io.py | 3 +- sensor/models.py | 35 +++--- sensor/views.py | 4 +- tests/globals.py | 17 +++ tests/sensor/__snapshots__/test_io.ambr | 5 +- tests/sensor/__snapshots__/test_models.ambr | 4 + tests/sensor/__snapshots__/test_views.ambr | 4 + .../api/__snapshots__/test_viewsets.ambr | 7 ++ tests/sensor/api/test_viewsets.py | 112 ++++++++++++++++++ tests/sensor/test_models.py | 19 +-- tests/sensor/test_views.py | 87 ++++++++++++++ 14 files changed, 260 insertions(+), 47 deletions(-) create mode 100644 tests/globals.py create mode 100644 tests/sensor/__snapshots__/test_models.ambr create mode 100644 tests/sensor/__snapshots__/test_views.ambr create mode 100644 tests/sensor/api/__snapshots__/test_viewsets.ambr create mode 100644 tests/sensor/api/test_viewsets.py create mode 100644 tests/sensor/test_views.py diff --git a/sensor/api/serializers.py b/sensor/api/serializers.py index 6d28f5d..2192a23 100644 --- a/sensor/api/serializers.py +++ b/sensor/api/serializers.py @@ -1,9 +1,15 @@ from rest_framework import serializers from ..models import File +from ..models import FileType class FileSerializer(serializers.ModelSerializer): + + type = serializers.SlugRelatedField( + slug_field="name", queryset=FileType.objects.all() + ) + class Meta: model = File fields = '__all__' diff --git a/sensor/api/viewsets.py b/sensor/api/viewsets.py index 70294fd..7bec629 100644 --- a/sensor/api/viewsets.py +++ b/sensor/api/viewsets.py @@ -5,8 +5,5 @@ class FileViewSet(viewsets.ModelViewSet): - """ - This viewset automatically provides `list` and `retrieve` actions. - """ queryset = File.objects.all() serializer_class = FileSerializer \ No newline at end of file diff --git a/sensor/api_urls.py b/sensor/api_urls.py index 2ab5850..b85d993 100644 --- a/sensor/api_urls.py +++ b/sensor/api_urls.py @@ -26,7 +26,6 @@ def api_root(request, format=None): return Response({ 'files': reverse('api:sensor:file-list', request=request, format=format), - 'echo': reverse('api:sensor:echo', request=request, format=format), }) diff --git a/sensor/io.py b/sensor/io.py index c1d5525..65d820e 100644 --- a/sensor/io.py +++ b/sensor/io.py @@ -28,9 +28,10 @@ def validate_datetime_fieldnames_in_lines( datetime_fieldnames: typing.Iterable[str], ) -> None: + split_lines = yield_split_lines(lines=lines, encoding=encoding, delimiter=delimiter) fieldnames = None - for line in yield_split_lines(lines=lines, encoding=encoding, delimiter=delimiter): + for line in split_lines: if set(datetime_fieldnames).issubset(set(line)): fieldnames = line break diff --git a/sensor/models.py b/sensor/models.py index e6a50dc..5929bee 100644 --- a/sensor/models.py +++ b/sensor/models.py @@ -7,6 +7,7 @@ from django.contrib.postgres.fields import ArrayField from django.db import models from django.db import transaction +from django.core.exceptions import ValidationError from .io import validate_datetime_fieldnames_in_lines from .io import yield_readings_in_narrow_format @@ -101,31 +102,30 @@ class File(models.Model): hash = models.TextField(blank=True, null=True) def clean(self): - # NOTE: automatically called by Django Forms & DRF Serializer Validate Method - with self.file.open(mode="rb") as f: - validate_datetime_fieldnames_in_lines( - lines=f, - encoding=self.type.encoding, - delimiter=self.type.delimiter, - datetime_fieldnames=self.type.datetime_fieldnames, - ) - def import_to_db(self): + if self.type is None: + raise ValidationError("File type must be specified!") - if not self.type: - message = ( - "Please define this file's type" - + " before attempting to parse it" - + " so the file's `encoding`, `delimiter`, `datetime_fieldnames`" - + " etc are defined!" - ) - raise ValueError(message) + # NOTE: This file is automatically closed upon saving a model instance + # ... each time a file is read the file pointer must be reset to enable rereads + f = self.file.open(mode="rb") + + # NOTE: automatically called by Django Forms & DRF Serializer Validate Method + validate_datetime_fieldnames_in_lines( + lines=f, + encoding=self.type.encoding, + delimiter=self.type.delimiter, + datetime_fieldnames=self.type.datetime_fieldnames, + ) + self.file.seek(0) + def import_to_db(self): with self.file.open(mode="rb") as f: reading_objs = ( Reading( + file=self, timestamp=r["timestamp"], sensor_name=r["sensor_name"], reading=r["reading"] @@ -150,7 +150,6 @@ def import_to_db(self): Reading.objects.bulk_create(batch, batch_size) except Exception as e: - breakpoint() self.parsed_at = None self.parse_error = str(e) self.save() diff --git a/sensor/views.py b/sensor/views.py index 74c24ae..cbf89c7 100644 --- a/sensor/views.py +++ b/sensor/views.py @@ -12,7 +12,7 @@ def create_file_type(request): form.save() return HttpResponse("File type was created") else: - return HttpResponse("File type creation failed") + return HttpResponse(f"File type creation failed: {form.errors}") else: form = FileTypeForm() return render(request, "create_file_type.html", {"form": form}) @@ -25,7 +25,7 @@ def upload_file(request): form.save() return HttpResponse("File upload was successful") else: - return HttpResponse("File upload failed") + return HttpResponse(f"File type creation failed: {form.errors}") else: form = FileForm() return render(request, "upload_file.html", {"form": form}) diff --git a/tests/globals.py b/tests/globals.py new file mode 100644 index 0000000..1e761ea --- /dev/null +++ b/tests/globals.py @@ -0,0 +1,17 @@ +SOURCES = [ + { + "lines": [ + b"Lat=0 Lon=0 Hub-Height=160 Timezone=00.0 Terrain-Height=0.0", + b"Computed at 100 m resolution", + b" ", + b"YYYYMMDD HHMM M(m/s) D(deg) SD(m/s) DSD(deg) Gust3s(m/s) T(C) PRE(hPa) RiNumber VertM(m/s)", + b"20151222 0000 20.54 211.0 1.22 0.3 21.00 11.9 992.8 0.15 0.18", + b"20151222 0010 21.02 212.2 2.55 0.6 21.35 11.8 992.7 0.29 -0.09", + ], + "encoding": "utf-8", + "delimiter": "\s+", + "datetime_fieldnames": ["YYYYMMDD", "HHMM"], + "datetime_formats": [r"%Y%m%d %H%M"], + "na_values": ["NAN"], + } +] diff --git a/tests/sensor/__snapshots__/test_io.ambr b/tests/sensor/__snapshots__/test_io.ambr index d281c34..666a676 100644 --- a/tests/sensor/__snapshots__/test_io.ambr +++ b/tests/sensor/__snapshots__/test_io.ambr @@ -1,8 +1,5 @@ # serializer version: 1 -# name: test_import_to_db[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0] - , , , , , , , , , , , , , , , , , , ]> -# --- -# name: test_yield_readings[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0] +# name: test_yield_readings_in_narrow_format[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0] list([ dict({ 'reading': '20.54', diff --git a/tests/sensor/__snapshots__/test_models.ambr b/tests/sensor/__snapshots__/test_models.ambr new file mode 100644 index 0000000..b770c5e --- /dev/null +++ b/tests/sensor/__snapshots__/test_models.ambr @@ -0,0 +1,4 @@ +# serializer version: 1 +# name: test_import_to_db[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0] + , , , , , , , , , , , , , , , , , , ]> +# --- diff --git a/tests/sensor/__snapshots__/test_views.ambr b/tests/sensor/__snapshots__/test_views.ambr new file mode 100644 index 0000000..74e0cf0 --- /dev/null +++ b/tests/sensor/__snapshots__/test_views.ambr @@ -0,0 +1,4 @@ +# serializer version: 1 +# name: TestUploadFile.test_cannot_upload_an_invalid_file[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0] + b'File type creation failed:
  • __all__
    • No `datetime_fieldnames` ['YYYYMMDD', 'HHMM'] found!
' +# --- diff --git a/tests/sensor/api/__snapshots__/test_viewsets.ambr b/tests/sensor/api/__snapshots__/test_viewsets.ambr new file mode 100644 index 0000000..3daee02 --- /dev/null +++ b/tests/sensor/api/__snapshots__/test_viewsets.ambr @@ -0,0 +1,7 @@ +# serializer version: 1 +# name: TestUploadFile.test_cannot_upload_a_missing_file_type[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0] + b'{"type":["This field may not be null."]}' +# --- +# name: TestUploadFile.test_cannot_upload_an_invalid_file[lines0-utf-8-\\s+-datetime_fieldnames0-datetime_formats0-na_values0] + b'{"non_field_errors":["No `datetime_fieldnames` [\'YYYYMMDD\', \'HHMM\'] found!"]}' +# --- diff --git a/tests/sensor/api/test_viewsets.py b/tests/sensor/api/test_viewsets.py new file mode 100644 index 0000000..4a14b4b --- /dev/null +++ b/tests/sensor/api/test_viewsets.py @@ -0,0 +1,112 @@ +from http import HTTPStatus + +from django.core.files.uploadedfile import SimpleUploadedFile +import pytest +from rest_framework.reverse import reverse + +from sensor.models import FileType +from tests.globals import SOURCES + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "lines,encoding,delimiter,datetime_fieldnames,datetime_formats,na_values", + [ + ( + source["lines"], + source["encoding"], + source["delimiter"], + source["datetime_fieldnames"], + source["datetime_formats"], + source["na_values"], + ) + for source in SOURCES + ] +) +class TestUploadFile(): + + def test_can_upload_a_valid_payload( + self, + client, + lines, + encoding, + delimiter, + datetime_fieldnames, + datetime_formats, + na_values, + ): + file_type = FileType.objects.create( + name="type", + encoding=encoding, + delimiter=delimiter, + datetime_fieldnames=datetime_fieldnames, + datetime_formats=datetime_formats, + na_values=na_values, + ) + file = SimpleUploadedFile( + name="sensor-readings.txt", content=b"\n".join(l for l in lines), + ) + url = reverse("api:sensor:file-list") + + response = client.post( + url, + {"file": file, "type": file_type.name} + ) + + assert response.status_code == HTTPStatus.CREATED + + def test_cannot_upload_an_invalid_file( + self, + client, + lines, + encoding, + delimiter, + datetime_fieldnames, + datetime_formats, + na_values, + snapshot, + ): + file_type = FileType.objects.create( + name="type", + encoding=encoding, + delimiter=delimiter, + datetime_fieldnames=datetime_fieldnames, + datetime_formats=datetime_formats, + na_values=na_values, + ) + invalid_file = SimpleUploadedFile( + name="sensor-readings.txt", content=b"I am invalid!",) + url = reverse("api:sensor:file-list") + + response = client.post( + url, + {"file": invalid_file, "type": file_type.name} + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.content == snapshot + + + def test_cannot_upload_a_missing_file_type( + self, + client, + lines, + encoding, + delimiter, + datetime_fieldnames, + datetime_formats, + na_values, + snapshot, + ): + file = SimpleUploadedFile( + name="sensor-readings.txt", content=b"\n".join(l for l in lines), + ) + url = reverse("api:sensor:file-list") + + response = client.post( + url, + {"file": file, "type": ""} + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.content == snapshot diff --git a/tests/sensor/test_models.py b/tests/sensor/test_models.py index 44675e2..59203d4 100644 --- a/tests/sensor/test_models.py +++ b/tests/sensor/test_models.py @@ -5,24 +5,7 @@ from sensor.models import FileType from sensor.models import Reading - -SOURCES = [ - { - "lines": [ - b"Lat=0 Lon=0 Hub-Height=160 Timezone=00.0 Terrain-Height=0.0", - b"Computed at 100 m resolution", - b" ", - b"YYYYMMDD HHMM M(m/s) D(deg) SD(m/s) DSD(deg) Gust3s(m/s) T(C) PRE(hPa) RiNumber VertM(m/s)", - b"20151222 0000 20.54 211.0 1.22 0.3 21.00 11.9 992.8 0.15 0.18", - b"20151222 0010 21.02 212.2 2.55 0.6 21.35 11.8 992.7 0.29 -0.09", - ], - "encoding": "utf-8", - "delimiter": "\s+", - "datetime_fieldnames": ["YYYYMMDD", "HHMM"], - "datetime_formats": [r"%Y%m%d %H%M"], - "na_values": ["NAN"], - } -] +from tests.globals import SOURCES @pytest.mark.django_db diff --git a/tests/sensor/test_views.py b/tests/sensor/test_views.py new file mode 100644 index 0000000..06977a7 --- /dev/null +++ b/tests/sensor/test_views.py @@ -0,0 +1,87 @@ +from http import HTTPStatus + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse +import pytest + +from sensor.models import FileType +from tests.globals import SOURCES + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "lines,encoding,delimiter,datetime_fieldnames,datetime_formats,na_values", + [ + ( + source["lines"], + source["encoding"], + source["delimiter"], + source["datetime_fieldnames"], + source["datetime_formats"], + source["na_values"], + ) + for source in SOURCES + ] +) +class TestUploadFile(): + + def test_can_upload_a_valid_payload( + self, + client, + lines, + encoding, + delimiter, + datetime_fieldnames, + datetime_formats, + na_values, + ): + file_type = FileType.objects.create( + name="type", + encoding=encoding, + delimiter=delimiter, + datetime_fieldnames=datetime_fieldnames, + datetime_formats=datetime_formats, + na_values=na_values, + ) + file = SimpleUploadedFile( + name="sensor-readings.txt", content=b"\n".join(l for l in lines), + ) + url = reverse("sensor:upload-file") + + response = client.post( + url, + {"file": file, "type": file_type.id} + ) + + assert response.status_code == HTTPStatus.OK + + def test_cannot_upload_an_invalid_file( + self, + client, + lines, + encoding, + delimiter, + datetime_fieldnames, + datetime_formats, + na_values, + snapshot, + ): + file_type = FileType.objects.create( + name="type", + encoding=encoding, + delimiter=delimiter, + datetime_fieldnames=datetime_fieldnames, + datetime_formats=datetime_formats, + na_values=na_values, + ) + invalid_file = SimpleUploadedFile( + name="sensor-readings.txt", content=b"I am invalid!",) + url = reverse("sensor:upload-file") + + response = client.post( + url, + {"file": invalid_file, "type": file_type.id} + ) + + assert response.status_code == HTTPStatus.OK + assert response.content == snapshot