diff --git a/core/urls.py b/core/urls.py
index 054603b..53894ac 100644
--- a/core/urls.py
+++ b/core/urls.py
@@ -21,7 +21,7 @@
urlpatterns = [
- path('', lambda request: redirect('sensor/')),
+ path('', lambda request: redirect('sensor:root')),
path('admin/', admin.site.urls),
path('api/', include('core.api_urls')),
diff --git a/sensor/migrations/0003_filetype.py b/sensor/migrations/0003_filetype.py
index 524a291..95f752b 100644
--- a/sensor/migrations/0003_filetype.py
+++ b/sensor/migrations/0003_filetype.py
@@ -16,11 +16,11 @@ class Migration(migrations.Migration):
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField()),
- ('na_values', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=10), default=['NaN'], help_text='A list of strings to recognise as empty values.
\n\n Default: ["NaN"]
\n\n Note: "" is also included by default
\n\n Example: ["NAN", "-9999", "-9999.0"]\n', size=None)),
- ('delimiter', models.CharField(default=',', help_text='The character used to separate fields in the file.
\n\n Default: ","
\n\n Examples: "," or ";" or "\\s+" for whitespace or "\\t" for tabs\n', max_length=5)),
- ('datetime_fieldnames', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=50), default=['Tmstamp'], help_text='A list of datetime field names.
\n\n Examples:
\n\n 1) Data has a single datetime field named "Tmstamp" which has values like\n \'2021-06-29 00:00:00.000\': ["Tmstamp"]
\n\n 2) Data has two datetime fields named "Date" and "Time" which have values\n like \'01.01.1999\' and \'00:00\' respectively: ["Date","Time"]
\n', size=None)),
- ('encoding', models.CharField(default='utf-8', help_text='The encoding of the file.
\n\n Default: "utf-8"
\n\n Examples: utf-8 or latin-1 or cp1252\n', max_length=25)),
- ('datetime_formats', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=25), default=['%Y-%m-%d %H:%M:%S'], help_text='The datetime format of `datetime_columns`.
\n\n See https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes\n for format codes\n\n Default: "%Y-%m-%d %H:%M:%S"
\n\n Examples: "%Y-%m-%d %H:%M:%S" for "2021-03-01 00:00:00"\n', size=None)),
+ ('na_values', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=10), default=['NaN'], help_text='A list of strings to recognise as empty values.\n\n Default: ["NaN"]\n\n Note: "" is also included by default\n\n Example: ["NAN", "-9999", "-9999.0"]\n', size=None)),
+ ('delimiter', models.CharField(default=',', help_text='The character used to separate fields in the file.\n\n Default: ","\n\n Examples: "," or ";" or "\\s+" for whitespace or "\\t" for tabs\n', max_length=5)),
+ ('datetime_fieldnames', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=50), default=['Tmstamp'], help_text='A list of datetime field names.\n\n Examples:\n\n 1) Data has a single datetime field named "Tmstamp" which has values like\n \'2021-06-29 00:00:00.000\': ["Tmstamp"]\n\n 2) Data has two datetime fields named "Date" and "Time" which have values\n like \'01.01.1999\' and \'00:00\' respectively: ["Date","Time"]\n', size=None)),
+ ('encoding', models.CharField(default='utf-8', help_text='The encoding of the file.\n\n Default: "utf-8"\n\n Examples: utf-8 or latin-1 or cp1252\n', max_length=25)),
+ ('datetime_formats', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=25), default=['%Y-%m-%d %H:%M:%S'], help_text='The datetime format of `datetime_columns`.\n\n See https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes\n for format codes\n\n Default: "%Y-%m-%d %H:%M:%S"\n\n Examples: "%Y-%m-%d %H:%M:%S" for "2021-03-01 00:00:00"\n', size=None)),
],
),
]
diff --git a/sensor/migrations/0005_source_alter_file_parsed_at_and_more.py b/sensor/migrations/0005_source_alter_file_parsed_at_and_more.py
deleted file mode 100644
index 8b358ad..0000000
--- a/sensor/migrations/0005_source_alter_file_parsed_at_and_more.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Generated by Django 5.0 on 2024-01-05 16:33
-
-import django.contrib.postgres.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('sensor', '0004_file_type'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Source',
- fields=[
- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('name', models.TextField()),
- ],
- ),
- migrations.AlterField(
- model_name='file',
- name='parsed_at',
- field=models.DateTimeField(default=None),
- preserve_default=False,
- ),
- migrations.AlterField(
- model_name='filetype',
- name='datetime_fieldnames',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=50), default=['Tmstamp'], help_text='A list of datetime field names.\n\n Examples:\n\n 1) Data has a single datetime field named "Tmstamp" which has values like\n \'2021-06-29 00:00:00.000\': ["Tmstamp"]\n\n 2) Data has two datetime fields named "Date" and "Time" which have values\n like \'01.01.1999\' and \'00:00\' respectively: ["Date","Time"]\n', size=None),
- ),
- migrations.AlterField(
- model_name='filetype',
- name='datetime_formats',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=25), default=['%Y-%m-%d %H:%M:%S'], help_text='The datetime format of `datetime_columns`.\n\n See https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes\n for format codes\n\n Default: "%Y-%m-%d %H:%M:%S"\n\n Examples: "%Y-%m-%d %H:%M:%S" for "2021-03-01 00:00:00"\n', size=None),
- ),
- migrations.AlterField(
- model_name='filetype',
- name='delimiter',
- field=models.CharField(default=',', help_text='The character used to separate fields in the file.\n\n Default: ","\n\n Examples: "," or ";" or "\\s+" for whitespace or "\\t" for tabs\n', max_length=5),
- ),
- migrations.AlterField(
- model_name='filetype',
- name='encoding',
- field=models.CharField(default='utf-8', help_text='The encoding of the file.\n\n Default: "utf-8"\n\n Examples: utf-8 or latin-1 or cp1252\n', max_length=25),
- ),
- migrations.AlterField(
- model_name='filetype',
- name='na_values',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=10), default=['NaN'], help_text='A list of strings to recognise as empty values.\n\n Default: ["NaN"]\n\n Note: "" is also included by default\n\n Example: ["NAN", "-9999", "-9999.0"]\n', size=None),
- ),
- ]
diff --git a/sensor/migrations/0005_source_alter_filetype_datetime_fieldnames_and_more.py b/sensor/migrations/0005_source_alter_filetype_datetime_fieldnames_and_more.py
new file mode 100644
index 0000000..9032b5d
--- /dev/null
+++ b/sensor/migrations/0005_source_alter_filetype_datetime_fieldnames_and_more.py
@@ -0,0 +1,21 @@
+# Generated by Django 5.0 on 2024-01-05 18:16
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('sensor', '0004_file_type'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Source',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.TextField()),
+ ],
+ )
+ ]
diff --git a/sensor/models.py b/sensor/models.py
index 51baea7..e6a50dc 100644
--- a/sensor/models.py
+++ b/sensor/models.py
@@ -13,10 +13,12 @@
class Source(models.Model):
+
name = models.TextField()
class FileType(models.Model):
+
name = models.TextField()
na_values = ArrayField(
base_field=models.CharField(max_length=10),
@@ -90,10 +92,11 @@ class FileType(models.Model):
class File(models.Model):
+
file = models.FileField(upload_to="readings/", blank=False, null=False)
uploaded_at = models.DateTimeField(auto_now_add=True)
type = models.ForeignKey(FileType, on_delete=models.RESTRICT)
- parsed_at = models.DateTimeField(blank=False, null=False)
+ parsed_at = models.DateTimeField(blank=True, null=True)
parse_error = models.TextField(blank=True, null=True)
hash = models.TextField(blank=True, null=True)
@@ -120,6 +123,7 @@ def import_to_db(self):
with self.file.open(mode="rb") as f:
+
reading_objs = (
Reading(
timestamp=r["timestamp"],
@@ -135,29 +139,31 @@ def import_to_db(self):
)
)
- batch_size = 1_000
+ batch_size = 1_000
- try:
- with transaction.atomic():
- while True:
- batch = list(islice(reading_objs, batch_size))
- if not batch:
- break
- Reading.objects.bulk_create(batch, batch_size)
-
- except Exception as e:
- self.parsed_at = None
- self.parse_error = str(e)
- self.save()
- raise e
-
- else:
- self.parsed_at = datetime.now(timezone.utc)
- self.parse_error = None
- self.save()
+ try:
+ with transaction.atomic():
+ while True:
+ batch = list(islice(reading_objs, batch_size))
+ if not batch:
+ break
+ Reading.objects.bulk_create(batch, batch_size)
+
+ except Exception as e:
+ breakpoint()
+ self.parsed_at = None
+ self.parse_error = str(e)
+ self.save()
+ raise e
+
+ else:
+ self.parsed_at = datetime.now(timezone.utc)
+ self.parse_error = None
+ self.save()
class Reading(models.Model):
+
file = models.ForeignKey(File, on_delete=models.RESTRICT)
timestamp = models.DateTimeField(blank=False, null=False, primary_key=True)
sensor_name = models.TextField(blank=False, null=False)
diff --git a/sensor/urls.py b/sensor/urls.py
index d302ce7..04a4336 100644
--- a/sensor/urls.py
+++ b/sensor/urls.py
@@ -1,4 +1,3 @@
-from django.shortcuts import redirect
from django.urls import path
from . import views
@@ -8,7 +7,7 @@
urlpatterns = [
- path('', views.index),
+ path('', views.index, name="root"),
path('create-file-type/', views.create_file_type, name="create-file-type"),
path('upload-file/', views.upload_file, name="upload-file"),
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..e71c9da
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,10 @@
+import pytest
+from django.conf import settings
+
+
+@pytest.fixture(scope='function', autouse=True)
+def media_root(tmp_path):
+ original_media_root = settings.MEDIA_ROOT
+ settings.MEDIA_ROOT = tmp_path / 'media'
+ yield settings.MEDIA_ROOT
+ settings.MEDIA_ROOT = original_media_root
diff --git a/tests/sensor/test_models.py b/tests/sensor/test_models.py
index c998b18..44675e2 100644
--- a/tests/sensor/test_models.py
+++ b/tests/sensor/test_models.py
@@ -59,8 +59,9 @@ def test_import_to_db(
)
file = ContentFile(b"\n".join(l for l in lines), name="sensor-readings.txt")
file_obj = File(file=file, type=file_type_obj)
+ file_obj.save()
- file_obj.import_to_db(file_obj)
+ file_obj.import_to_db()
output = Reading.objects.all()
assert output == snapshot