Add required fields and prevent null values by converting to empty st…

…rings

datahub/company_activity/tasks/ingest_stova_events.py

@@ -51,9 +51,48 @@ def _should_process_record(self, record: dict) -> bool:
 
         return True
 
+    @staticmethod
+    def _required_fields() -> list:
+        """
+        Returns a list of fields required for to make a StovaEvent a Data Hub Event.
+        Any fields listed here but not provided by Stova will be rejected from ingestion.
+
+        :return: Required fields to save a StovaEvent.
+        """
+        return [
+            'stova_event_id',
+            'name',
+            'location_address1',
+            'location_city',
+        ]
+
+    @staticmethod
+    def _convert_fields_from_null_to_blank(values: dict) -> dict:
+        """
+        Coverts values from the stova record which could be null into empty strings for saving
+        as a Data Hub event.
+
+        :param values: A single Stova Event record from an S3 bucket.
+        :return: A single Stova Event record with null/None values replaced with empty strings.
+        """
+        fields_required_as_blank = [
+            'location_address2',
+            'location_address3',
+            'location_state',
+            'location_postcode',
+            'description',
+        ]
+
+        for field in fields_required_as_blank:
+            if values[field] is None:
+                values[field] = ''
+
+        return values
+
     def _process_record(self, record: dict) -> None:
         """Saves an event from Stova from the S3 bucket into a `StovaEvent`"""
         stova_event_id = record.get('id')
+
         values = {
             'stova_event_id': record.get('id'),
             'url': record.get('url', ''),
@@ -90,6 +129,17 @@ def _process_record(self, record: dict) -> None:
             'standard_currency': record.get('standard_currency', ''),
         }
 
+        required_fields = self._required_fields()
+        for field in required_fields:
+            if values[field] is None or values[field] == '':
+                logger.info(
+                    f'Stova Event with id {stova_event_id} does not have required field {field}. '
+                    'This stova event will not be processed into Data Hub.',
+                )
+                return
+
+        self._convert_fields_from_null_to_blank(values)
+
         try:
             stova_event = StovaEvent(**values)
             # Raises Validation errors if there are any with the field/s which errored

datahub/company_activity/tests/test_tasks/test_stova_ingestion_task.py

@@ -253,3 +253,48 @@ def test_stova_event_ingestion_handles_unexpected_fields(self, caplog, test_base
             ) in caplog.text
 
             assert 'approval_required' in caplog.text
+
+    @pytest.mark.django_db
+    def test_stova_event_ingestion_rejects_event_if_missing_required_field(
+        self, caplog, test_base_stova_event,
+    ):
+        """
+        Some fields are required by Data Hub events, if a Stova Event does not provide these fields
+        the stova event will not be ingested.
+        """
+        s3_processor_mock = mock.Mock()
+        task = StovaEventIngestionTask('dummy-prefix', s3_processor_mock)
+
+        data = test_base_stova_event
+        stova_event_id = data.get('id')
+
+        # This is required so a Stova Event can be saved as a Data Hub event.
+        data['name'] = None
+
+        with caplog.at_level(logging.INFO):
+            task._process_record(data)
+            assert (
+                f'Stova Event with id {stova_event_id} does not have required field name. '
+                'This stova event will not be processed into Data Hub.'
+            ) in caplog.text
+
+        assert StovaEvent.objects.count() == 0
+
+    @pytest.mark.django_db
+    def test_stova_event_ingestion_converts_null_fields_to_empty_string(
+        self, test_base_stova_event,
+    ):
+        """
+        Some fields are required to be an empty string by Data Hub Events, they do not accept
+        null values.
+        """
+        s3_processor_mock = mock.Mock()
+        task = StovaEventIngestionTask('dummy-prefix', s3_processor_mock)
+
+        data = test_base_stova_event
+        # This must be empty string to be saved, test it gets converted from None
+        data['description'] = None
+        task._process_record(data)
+
+        assert StovaEvent.objects.count() == 1
+        assert StovaEvent.objects.first().description == ''