Add postcode data ingestion job

cron-scheduler.py

@@ -39,6 +39,7 @@
     EVERY_THREE_AM,
     EVERY_THREE_AM_ON_TWENTY_EIGHTH_EACH_MONTH,
     EVERY_TWO_AM,
+    EVERY_WEEK,
     HALF_DAY_IN_SECONDS,
     ONE_HOUR_IN_SECONDS,
 )
@@ -57,6 +58,9 @@
     schedule_refresh_gross_value_added_value_for_fdi_investment_projects,
 )
 from datahub.investment_lead.tasks.ingest_eyb_triage import eyb_triage_identification_task
+from datahub.metadata.tasks import (
+    postcode_data_identification_task,
+)
 from datahub.omis.payment.tasks import refresh_pending_payment_gateway_sessions
 from datahub.reminder.migration_tasks import run_ita_users_migration, run_post_users_migration
 from datahub.reminder.tasks import (
@@ -154,6 +158,11 @@ def schedule_jobs():
         cron=EVERY_HOUR,
         description='Identify new Stova attendee objects and schedule their ingestion',
     )
+    job_scheduler(
+        function=postcode_data_identification_task,
+        cron=EVERY_WEEK,
+        description='Identify new Postcode objects and schedule their ingestion',
+    )
 
     if settings.ENABLE_ESTIMATED_LAND_DATE_REMINDERS:
         job_scheduler(

datahub/company_activity/tasks/ingest_stova_attendees.py

@@ -21,7 +21,7 @@
 def stova_attendee_identification_task() -> None:
     """Identifies the most recent file to be ingested and schedules a task to ingest it"""
     logger.info('Stova attendee identification task started.')
-    identification_task = StovaAttendeeIndentificationTask(prefix=STOVA_ATTENDEE_PREFIX)
+    identification_task = StovaAttendeeIdentificationTask(prefix=STOVA_ATTENDEE_PREFIX)
     identification_task.identify_new_objects(stova_attendee_ingestion_task)
     logger.info('Stova attendee identification task finished.')
 
@@ -37,7 +37,7 @@ def stova_attendee_ingestion_task(object_key: str) -> None:
     logger.info(f'Stova attendee ingestion task finished for file {object_key}.')
 
 
-class StovaAttendeeIndentificationTask(BaseObjectIdentificationTask):
+class StovaAttendeeIdentificationTask(BaseObjectIdentificationTask):
     pass
 
 

datahub/company_activity/tasks/ingest_stova_events.py

@@ -15,7 +15,7 @@
 def stova_event_identification_task() -> None:
     """Identifies the most recent file to be ingested and schedules a task to ingest it"""
     logger.info('Stova event identification task started.')
-    identification_task = StovaEventIndentificationTask(prefix=STOVA_EVENT_PREFIX)
+    identification_task = StovaEventIdentificationTask(prefix=STOVA_EVENT_PREFIX)
     identification_task.identify_new_objects(stova_event_ingestion_task)
     logger.info('Stova event identification task finished.')
 
@@ -31,7 +31,7 @@ def stova_event_ingestion_task(object_key: str) -> None:
     logger.info(f'Stova event ingestion task finished for file {object_key}.')
 
 
-class StovaEventIndentificationTask(BaseObjectIdentificationTask):
+class StovaEventIdentificationTask(BaseObjectIdentificationTask):
     pass
 
 

datahub/core/queues/constants.py

@@ -26,3 +26,5 @@
 THIRTY_MINUTES_IN_SECONDS = 60 * 30
 ONE_HOUR_IN_SECONDS = 60 * 60
 HALF_DAY_IN_SECONDS = 12 * ONE_HOUR_IN_SECONDS
+
+EVERY_WEEK = '0 0 * * MON'

datahub/metadata/constants.py

@@ -0,0 +1,9 @@
+import environ
+
+from datahub.ingest.constants import PREFIX
+
+
+env = environ.Env()
+
+
+POSTCODE_DATA_PREFIX = f'{PREFIX}ExportPostcodeDirectory/'

datahub/metadata/metadata.py

@@ -6,6 +6,7 @@
     CountrySerializer,
     ExchangeRateSerializer,
     InvestmentProjectStageSerializer,
+    PostcodeDataSerializer,
     SectorSerializer,
     ServiceSerializer,
     TeamSerializer,
@@ -104,3 +105,8 @@
 )
 registry.register(metadata_id='fdi-value', model=models.FDIValue)
 registry.register(metadata_id='export-barrier', model=models.ExportBarrierType)
+registry.register(
+    metadata_id='postcode-data',
+    model=models.PostcodeData,
+    serializer=PostcodeDataSerializer,
+)

datahub/metadata/migrations/0090_postcodedata.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.17 on 2025-02-19 14:04
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('metadata', '0089_add_stova_serivce'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PostcodeData',
+            fields=[
+                ('disabled_on', models.DateTimeField(blank=True, null=True)),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('name', models.TextField(blank=True)),
+                ('postcode', models.CharField(max_length=255)),
+                ('modified_on', models.DateTimeField(auto_now=True, null=True)),
+                ('publication_date', models.DateTimeField(blank=True, null=True)),
+                ('region', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='metadata.ukregion')),
+            ],
+            options={
+                'ordering': ('name',),
+                'abstract': False,
+            },
+        ),
+    ]

datahub/metadata/models.py

@@ -405,3 +405,18 @@ class FDIValue(BaseOrderedConstantModel):
 
 class ExportBarrierType(BaseOrderedConstantModel):
     """Export barrier type (used for company interactions)."""
+
+
+class PostcodeData(BaseConstantModel):
+    """Postcode data (for the manual addition of a company)."""
+
+    postcode = models.CharField(max_length=MAX_LENGTH)
+    modified_on = models.DateTimeField(auto_now=True, null=True)
+    postcode = models.ForeignKey(
+        UKRegion,
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name='+',
+    )
+    publication_date = models.DateTimeField(null=True, blank=True)

datahub/metadata/serializers.py

@@ -7,7 +7,7 @@
 from datahub.export_win.models import TeamType
 from datahub.interaction.models import ServiceAnswerOption, ServiceQuestion
 from datahub.metadata.models import (
-    Country, ExchangeRate, OverseasRegion, Service, TeamRole, UKRegion,
+    Country, ExchangeRate, OverseasRegion, PostcodeData, Service, TeamRole, UKRegion,
 )
 
 
@@ -127,3 +127,17 @@ class HVCSerializer(ConstantModelSerializer):
 
     campaign_id = serializers.ReadOnlyField()
     financial_year = serializers.ReadOnlyField()
+
+
+class PostcodeDataSerializer(ConstantModelSerializer):
+    """Postcode data serializer"""
+
+    id = serializers.UUIDField()
+    postcode = serializers.CharField()
+    modified_on = serializers.DateTimeField()
+    postcode_region = NestedRelatedField(UKRegion, read_only=True)
+    publication_date = serializers.DateTimeField()
+
+    class Meta:
+        model = PostcodeData
+        fields = '__all__'

datahub/metadata/tasks.py

@@ -0,0 +1,86 @@
+import logging
+
+from rest_framework import serializers
+
+from datahub.ingest.boto3 import S3ObjectProcessor
+from datahub.ingest.tasks import BaseObjectIdentificationTask, BaseObjectIngestionTask
+from datahub.metadata.constants import POSTCODE_DATA_PREFIX
+from datahub.metadata.models import PostcodeData
+from datahub.metadata.serializers import PostcodeDataSerializer
+
+
+logger = logging.getLogger(__name__)
+
+
+def postcode_data_identification_task() -> None:
+    logger.info('Postcode data identification task started...')
+    identification_task = PostcodeDataIdentificationTask(prefix=POSTCODE_DATA_PREFIX)
+    identification_task.identify_new_objects(postcode_data_ingestion_task)
+    logger.info('Postcode data identification task finished.')
+
+
+class PostcodeDataIdentificationTask(BaseObjectIdentificationTask):
+    """Class to identify new postcode data objects and determine if they should be ingested."""
+
+
+def postcode_data_ingestion_task(object_key: str) -> None:
+    logger.info('Postcode data ingestion task started...')
+    ingestion_task = PostcodeDataIngestionTask(
+        object_key=object_key,
+        s3_processor=S3ObjectProcessor(prefix=POSTCODE_DATA_PREFIX),
+        serializer_class=PostcodeDataSerializer,
+    )
+    ingestion_task.ingest_object()
+    logger.info('Postcode data ingestion task finished.')
+
+
+class PostcodeDataIngestionTask(BaseObjectIngestionTask):
+    """Class to ingest a postcode object from S3."""
+
+    def __init__(
+        self,
+        object_key: str,
+        s3_processor: S3ObjectProcessor,
+        serializer_class: serializers.Serializer,
+    ) -> None:
+        self.serializer_class = serializer_class
+        super().__init__(object_key, s3_processor)
+
+    existing_ids = []
+
+    def _should_process_record(self, record: dict) -> bool:
+        """Checks whether the record has already been ingested or not."""
+        if not self.existing_ids:
+            self.existing_ids = set(PostcodeData.objects.values_list(
+                'id', flat=True))
+
+        postcode_data_id = record.get('id')
+        if postcode_data_id in self.existing_ids:
+            logger.info(f'Record already exists for postcode_data_id: {postcode_data_id}')
+            return False
+
+        return True
+
+    def _process_record(self, record: dict) -> None:
+        """Processes a single record.
+
+        This method should take a single record, update an existing instance,
+        or create a new one, and return None.
+        """
+        serializer = self.serializer_class(data=record)
+        if serializer.is_valid():
+            primary_key = serializer.validated_data.pop('id')
+            queryset = PostcodeData.objects.filter(pk=primary_key)
+            instance, created = queryset.update_or_create(
+                pk=primary_key,
+                defaults=serializer.validated_data,
+            )
+            if created:
+                self.created_ids.append(str(instance.id))
+            else:
+                self.updated_ids.append(str(instance.id))
+        else:
+            self.errors.append({
+                'record': record,
+                'errors': serializer.errors,
+            })

datahub/metadata/test/factories.py

@@ -1,10 +1,16 @@
+import uuid
+from datetime import timezone
+
 from random import randrange, sample
 
 import factory
+from faker import Faker
 
 from datahub.core import constants
 from datahub.metadata.models import Service
 
+fake = Faker(locale='en_GB')
+
 
 class ServiceFactory(factory.django.DjangoModelFactory):
     """Service factory."""
@@ -119,3 +125,32 @@ class AdministrativeAreasFactory(factory.django.DjangoModelFactory):
 
     class Meta:
         model = 'metadata.AdministrativeArea'
+
+
+class PostcodeDataFactory(factory.django.DjangoModelFactory):
+    """Postcode data factory"""
+
+    postcode = factory.Faker('postcode')
+    modified_on = '2025-10-08T08:06:53+00:00'
+    postcode_region = factory.Faker('postcode_region')
+    publication_date = '2025-02-02T08:08:52+00:00'
+
+    class Meta:
+        model = 'metadata.PostcodeData'
+
+
+def postcode_data_record_faker(overrides: dict | None = None) -> dict:
+    data = {
+        'id': str(uuid.uuid4()),
+        'postcode': fake.postcode(),
+        'modified_on': fake.date_time_between(
+            start_date='-1y', tzinfo=timezone.utc,
+        ),
+        'publication_date': fake.date_time_between(
+            start_date='-1y', tzinfo=timezone.utc,
+        ),
+        'postcode_region': constants.UKRegion.london.name,
+    }
+    if overrides:
+        data.update(overrides)
+    return data