diff --git a/datahub/export_win/admin.py b/datahub/export_win/admin.py index 0358da024..33d313a43 100644 --- a/datahub/export_win/admin.py +++ b/datahub/export_win/admin.py @@ -176,9 +176,13 @@ class WinAdmin(BaseModelAdminMixin, VersionAdmin): ) search_fields = ( '=id', + 'computed_adviser_name', + # legacy field 'adviser_name', '=company__pk', 'lead_officer_adviser_name', + # legacy field + 'lead_officer_name', 'company__name', 'country__name', 'contact_name', @@ -239,7 +243,9 @@ class WinAdmin(BaseModelAdminMixin, VersionAdmin): def get_adviser(self, obj): """Return adviser as user with email.""" - return f'{obj.adviser} <{obj.adviser.email}>' + return f'{obj.adviser} <{obj.adviser.email}>' if obj.adviser else \ + f'{obj.adviser_name} <{obj.adviser_email_address}>' + get_adviser.short_description = 'User' def get_company(self, obj): @@ -283,7 +289,7 @@ def has_delete_permission(self, request, obj=None): def get_search_results(self, request, queryset, search_term): if search_term: queryset = queryset.annotate( - adviser_name=Concat( + computed_adviser_name=Concat( 'adviser__first_name', Value(' '), 'adviser__last_name', ), lead_officer_adviser_name=Concat( @@ -356,7 +362,7 @@ def has_change_permission(self, request, obj=None): class WinAdviserAdmin(BaseModelAdminMixin): """Admin for Win Adviser.""" - list_display = ('win', 'get_adviser_name', 'team_type', 'hq_team', 'location') + list_display = ('win', 'get_computed_adviser_name', 'team_type', 'hq_team', 'location') search_fields = ('win__id',) fieldsets = ( @@ -383,9 +389,9 @@ def get_queryset(self, request): queryset = super().get_queryset(request) return queryset.filter(win__is_deleted=False) - def get_adviser_name(self, obj): - """Return adviser name.""" - return obj.adviser.name + def get_computed_adviser_name(self, obj): + """Return computed adviser name.""" + return obj.adviser.name if obj.adviser else obj.name def delete_view(self, request, object_id, extra_context=None): """ @@ -399,6 +405,6 @@ def delete_view(self, request, object_id, extra_context=None): def has_change_permission(self, request, obj=None): return False - get_adviser_name.short_description = 'Name' + get_computed_adviser_name.short_description = 'Name' WinAdviser._meta.verbose_name_plural = 'Advisors' diff --git a/datahub/export_win/export_wins_api.py b/datahub/export_win/export_wins_api.py new file mode 100644 index 000000000..e536ba3c2 --- /dev/null +++ b/datahub/export_win/export_wins_api.py @@ -0,0 +1,75 @@ +from logging import getLogger + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from requests.exceptions import HTTPError, Timeout + +from datahub.company.export_wins_api import ( + ExportWinsAPIConnectionError, + ExportWinsAPIHTTPError, + ExportWinsAPITimeoutError, +) +from datahub.core.api_client import APIClient, HawkAuth +from datahub.core.exceptions import APIBadGatewayException + + +logger = getLogger(__name__) + + +def _fetch_page(api_client, source_url): + """ + Requests a page from given relative source_url. + """ + page = api_client.request('GET', source_url).json() + return page['results'], page['next'] + + +def get_legacy_export_wins_dataset(start_url): + if not all( + [ + settings.EXPORT_WINS_SERVICE_BASE_URL, + settings.EXPORT_WINS_HAWK_ID, + settings.EXPORT_WINS_HAWK_KEY, + ], + ): + raise ImproperlyConfigured('The all EXPORT_WINS_SERVICE* setting must be set') + + api_client = APIClient( + api_url=settings.EXPORT_WINS_SERVICE_BASE_URL, + auth=HawkAuth(settings.EXPORT_WINS_HAWK_ID, settings.EXPORT_WINS_HAWK_KEY), + raise_for_status=True, + default_timeout=settings.DEFAULT_SERVICE_TIMEOUT, + ) + + source_url = start_url + items = 0 + try: + while True: + results, next_url = _fetch_page(api_client, source_url) + + yield results + + items += len(results) + + if not next_url or 'source=E' in next_url: + break + + source_url = next_url.replace( + settings.EXPORT_WINS_SERVICE_BASE_URL, + '', + ) + + except APIBadGatewayException as exc: + error_message = 'Export Wins API service unavailable' + raise ExportWinsAPIConnectionError(error_message) from exc + except Timeout as exc: + error_message = 'Encountered a timeout interacting with Export Wins API' + raise ExportWinsAPITimeoutError(error_message) from exc + except HTTPError as exc: + error_message = ( + 'The Export Wins API returned an error status: ' + f'{exc.response.status_code}', + ) + raise ExportWinsAPIHTTPError(error_message) from exc + + logger.info(f'Legacy Export wins dataset {start_url} processed ({items} records).') diff --git a/datahub/export_win/legacy_migration.py b/datahub/export_win/legacy_migration.py new file mode 100644 index 000000000..bfd69fe44 --- /dev/null +++ b/datahub/export_win/legacy_migration.py @@ -0,0 +1,504 @@ +from datetime import datetime + +from dateutil.parser import parse + +from datahub.company.models import ( + Advisor, + Contact, + ExportExperience, +) +from datahub.core.utils import get_financial_year +from datahub.export_win.export_wins_api import get_legacy_export_wins_dataset +from datahub.export_win.models import ( + AssociatedProgramme, + Breakdown, + BreakdownType, + BusinessPotential, + CustomerResponse, + ExpectedValueRelation, + Experience, + HQTeamRegionOrPost, + HVC, + HVOProgrammes, + LegacyExportWinsToDataHubCompany, + MarketingSource, + Rating, + SupportType, + TeamType, + Win, + WinAdviser, + WinUKRegion, + WithoutOurSupport, +) +from datahub.export_win.utils import calculate_totals_for_export_win +from datahub.metadata.models import ( + Country, + Sector, +) +from datahub.metadata.query_utils import get_sector_name_subquery + + +def create_customer_response_from_legacy(win, item): + must_resolvers = { + 'access_to_contacts': resolve_legacy_field( + Rating, + 'confirmation__access_to_contacts', + ), + 'access_to_information': resolve_legacy_field( + Rating, + 'confirmation__access_to_information', + ), + 'developed_relationships': resolve_legacy_field( + Rating, + 'confirmation__developed_relationships', + ), + 'gained_confidence': resolve_legacy_field( + Rating, + 'confirmation__gained_confidence', + ), + 'improved_profile': resolve_legacy_field( + Rating, + 'confirmation__improved_profile', + ), + 'our_support': resolve_legacy_field( + Rating, + 'confirmation__our_support', + ), + 'overcame_problem': resolve_legacy_field( + Rating, + 'confirmation__overcame_problem', + ), + 'last_export': resolve_legacy_field( + Experience, + 'confirmation_last_export', + 'name', + ), + 'marketing_source': resolve_legacy_field( + MarketingSource, + 'confirmation_marketing_source', + 'name', + ), + 'expected_portion_without_help': resolve_legacy_field( + WithoutOurSupport, + 'confirmation_portion_without_help', + 'name', + ), + } + + field_mappings = { + 'confirmation__agree_with_win': 'agree_with_win', + 'confirmation__case_study_willing': 'case_study_willing', + 'confirmation__comments': 'comments', + 'confirmation__company_was_at_risk_of_not_exporting': + 'company_was_at_risk_of_not_exporting', + 'confirmation__has_enabled_expansion_into_existing_market': + 'has_enabled_expansion_into_existing_market', + 'confirmation__has_enabled_expansion_into_new_market': + 'has_enabled_expansion_into_new_market', + 'confirmation__has_explicit_export_plans': + 'has_explicit_export_plans', + 'confirmation__has_increased_exports_as_percent_of_turnover': + 'has_increased_exports_as_percent_of_turnover', + 'confirmation__interventions_were_prerequisite': + 'interventions_were_prerequisite', + 'confirmation__involved_state_enterprise': + 'involved_state_enterprise', + 'confirmation__other_marketing_source': + 'other_marketing_source', + 'confirmation__support_improved_speed': + 'support_improved_speed', + 'confirmation__created': 'responded_on', + 'confirmation__name': 'name', + } + + customer_response = {} + + for field, resolver in must_resolvers.items(): + resolved = resolver(item) + if isinstance(resolved, dict): + customer_response.update(resolved) + else: + customer_response.update( + {field: resolved}, + ) + for field, mapping in field_mappings.items(): + value = item.get(field) + customer_response.update( + **{mapping: value if value is not None else ''}, + ) + + return customer_response + + +def create_export_win_from_legacy(item): + must_resolvers = { + 'associated_programme': resolve_many_to_many( + AssociatedProgramme, + 'associated_programme_', + 5, + ), + 'business_potential': resolve_legacy_field( + BusinessPotential, + 'business_potential', + ), + 'company': resolve_company, + 'company_contacts': resolve_company_contact, + 'country': resolve_legacy_field( + Country, + 'country_name', + 'name', + ), + 'customer_location': resolve_legacy_field( + WinUKRegion, + 'customer_location', + ), + 'export_experience': resolve_legacy_field( + ExportExperience, + 'export_experience_display', + 'name', + ), + 'goods_vs_services': resolve_legacy_field( + ExpectedValueRelation, + 'goods_vs_services', + ), + 'hq_team': resolve_legacy_field( + HQTeamRegionOrPost, + 'hq_team', + ), + 'hvc': resolve_legacy_field( + HVC, + 'hvc', + ), + 'hvo_programme': resolve_legacy_field( + HVOProgrammes, + 'hvo_programme', + ), + 'team_type': resolve_legacy_field( + TeamType, + 'team_type', + ), + 'type_of_support': resolve_many_to_many( + SupportType, + 'type_of_support_', + 3, + ), + 'sector': resolve_legacy_field( + Sector, + 'sector_display', + 'name', + annotate={'name': get_sector_name_subquery()}, + ), + 'adviser': resolve_adviser, + 'lead_officer': resolve_lead_officer, + 'line_manager': resolve_line_manager, + 'migrated_on': lambda item, context: datetime.now(), + 'created_on': lambda item, context: parse(item['created']), + } + + # fields to be copied over directly + field_mappings = { + 'audit': 'audit', + 'business_type': 'business_type', + 'cdms_reference': 'cdms_reference', + 'complete': 'complete', + 'date': 'date', + 'description': 'description', + 'has_hvo_specialist_involvement': 'has_hvo_specialist_involvement', + 'is_e_exported': 'is_e_exported', + 'is_line_manager_confirmed': 'is_line_manager_confirmed', + 'is_personally_confirmed': 'is_personally_confirmed', + 'is_prosperity_fund_related': 'is_prosperity_fund_related', + 'name_of_customer': 'name_of_customer', + 'name_of_export': 'name_of_export', + 'other_official_email_address': 'other_official_email_address', + } + + win = { + 'id': item.get('id'), + } + + for field, resolver in must_resolvers.items(): + resolved = resolver(item, win) + if isinstance(resolved, dict): + win.update(resolved) + else: + win.update( + {field: resolved}, + ) + for field, mapping in field_mappings.items(): + value = item.get(field) + win.update( + **{mapping: value if value is not None else ''}, + ) + + return win + + +def migrate_legacy_win(item): + customer_response_item = { + key: item.pop(key) + for key in list(item.keys()) if key.startswith('confirmation_') + } + + win_data = create_export_win_from_legacy(item) + + many_to_many_fields = [ + 'associated_programme', + 'company_contacts', + 'type_of_support', + ] + many_to_many = { + field: win_data.pop(field) + for field in many_to_many_fields if field in win_data + } + win_id = win_data.pop('id') + win, created = Win.objects.update_or_create( + id=win_id, + defaults=win_data, + ) + if not created: + # The associated models need to be deleted + # to avoid duplicates when migrating them again + Breakdown.objects.filter(win_id=win.id).delete() + WinAdviser.objects.filter(win_id=win.id).delete() + win.created_on = win_data['created_on'] + win.save() + + for field_name, values in many_to_many.items(): + getattr(win, field_name).set(values) + customer_response_data = create_customer_response_from_legacy( + win, + customer_response_item, + ) + customer_response, _ = CustomerResponse.objects.update_or_create( + win_id=win_id, + defaults=customer_response_data, + ) + customer_response.created_on = win_data['created_on'] + customer_response.save() + return win + + +def create_breakdown_from_legacy(item): + win = Win.objects.get(id=item['win__id']) + breakdown_type = BreakdownType.objects.get(export_win_id=item['type']) + year = item['year'] - get_financial_year(win.date) + 1 + return { + 'win': win, + 'type': breakdown_type, + 'year': year, + 'value': int(item['value']), + } + + +def migrate_legacy_win_breakdown(item): + """Create breakdown from item.""" + breakdown_data = create_breakdown_from_legacy(item) + if not breakdown_data: + return None + + breakdown = Breakdown(**breakdown_data) + breakdown.save() + return breakdown + + +def update_legacy_win_totals(): + updated = 0 + for win in Win.objects.filter(migrated_on__isnull=False): + calc_total = calculate_totals_for_export_win(win) + win.total_expected_export_value = calc_total['total_export_value'] + win.total_expected_non_export_value = calc_total['total_non_export_value'] + win.total_expected_odi_value = calc_total['total_odi_value'] + win.save() + updated += 1 + return updated + + +def create_win_adviser_from_legacy(item): + hq_team = HQTeamRegionOrPost.objects.get(export_win_id=item['hq_team']) + team_type = TeamType.objects.get(export_win_id=item['team_type']) + + adviser_data = { + 'win_id': item['win__id'], + 'hq_team': hq_team, + 'team_type': team_type, + 'location': item['location'], + } + + parts = item.get('name').split() + # In case name is written as "Joe M. Doe" + first_name = parts[0] + last_name = parts[-1] + try: + adviser = Advisor.objects.get( + first_name__iexact=first_name.strip(), + last_name__iexact=last_name.strip(), + ) + adviser_data.update({ + 'adviser': adviser, + }) + except Advisor.DoesNotExist: + adviser_data.update({ + 'name': item['name'], + }) + return adviser_data + + +def migrate_legacy_win_adviser(item): + """Create win adviser from item.""" + adviser_data = create_win_adviser_from_legacy(item) + if not adviser_data: + return None + + adviser = WinAdviser(**adviser_data) + adviser.save() + return adviser + + +def resolve_legacy_field(model, source_field_name, lookup_field=None, annotate=None): + if not lookup_field: + lookup_field = 'export_win_id' + if not annotate: + annotate = {} + + def resolver(data, context=None): + field_value = data.get(source_field_name) + if field_value == '': + return None + try: + obj = model.objects.annotate(**annotate).get(**{lookup_field: field_value}) + return obj + except model.DoesNotExist: + return None + return resolver + + +def resolve_many_to_many(model, source_field_name, max_items): + def resolver(data, context=None): + resolvers = [ + resolve_legacy_field(model, f'{source_field_name}{i}') + for i in range(1, max_items + 1) + ] + objs = [ + obj for obj in ( + _resolver(data) for _resolver in resolvers + ) if obj + ] + + return objs + return resolver + + +def resolve_company(data, context=None): + try: + mapping = LegacyExportWinsToDataHubCompany.objects.get( + id=data.get('id'), + ) + return mapping.company + except LegacyExportWinsToDataHubCompany.DoesNotExist: + return { + 'company_name': data.get('company_name'), + } + + +def resolve_adviser(data, context=None): + try: + adviser = Advisor.objects.get( + contact_email__iexact=data.get('user__email').strip(), + ) + return adviser + except Advisor.DoesNotExist: + return { + 'adviser_name': data.get('user__name'), + 'adviser_email_address': data.get('user__email'), + } + + +def resolve_lead_officer(data, context=None): + try: + criteria = {} + email = data.get('lead_officer_email_address').strip() + if email != '': + criteria.update({ + 'contact_email__iexact': email, + }) + else: + parts = data.get('lead_officer_name').split() + # In case name is written as "Joe M. Doe" + first_name = parts[0] + last_name = parts[-1] + criteria.update({ + 'first_name__iexact': first_name.strip(), + 'last_name__iexact': last_name.strip(), + }) + adviser = Advisor.objects.get( + **criteria, + ) + return adviser + except Advisor.DoesNotExist: + return { + 'lead_officer_name': data.get('lead_officer_name'), + 'lead_officer_email_address': data.get('lead_officer_email_address'), + } + + +def resolve_line_manager(data, context=None): + try: + first_name, last_name = data.get('line_manager_name').split(' ') + adviser = Advisor.objects.get( + first_name__iexact=first_name.strip(), + last_name__iexact=last_name.strip(), + ) + return adviser + except Advisor.DoesNotExist: + return { + 'line_manager_name': data.get('line_manager_name'), + } + + +def _get_legacy_customer_info(data): + return { + 'customer_name': data['customer_name'], + 'customer_job_title': data['customer_job_title'], + 'customer_email_address': data['customer_email_address'], + } + + +def resolve_company_contact(data, context=None): + if 'company' in context: + parts = data['customer_name'].split() + # In case name is written as "Joe M. Doe" + first_name = parts[0] + last_name = parts[-1] + try: + contact = Contact.objects.get( + first_name__iexact=first_name, + last_name__iexact=last_name, + company=context['company'], + ) + return { + 'company_contacts': [contact], + } + except Contact.DoesNotExist: + pass + return _get_legacy_customer_info(data) + + +def migrate_all_legacy_wins(): + wins = 0 + for page in get_legacy_export_wins_dataset('/data-hub-wins'): + for legacy_win in page: + migrate_legacy_win(legacy_win) + wins += 1 + + for page in get_legacy_export_wins_dataset('/data-hub-breakdowns'): + for legacy_breakdown in page: + migrate_legacy_win_breakdown(legacy_breakdown) + + for page in get_legacy_export_wins_dataset('/data-hub-advisers'): + for legacy_adviser in page: + migrate_legacy_win_adviser(legacy_adviser) + + update_legacy_win_totals() + + return wins diff --git a/datahub/export_win/migrations/0039_win_adviser_email_address_win_adviser_name_and_more.py b/datahub/export_win/migrations/0039_win_adviser_email_address_win_adviser_name_and_more.py new file mode 100644 index 000000000..af7ba6de1 --- /dev/null +++ b/datahub/export_win/migrations/0039_win_adviser_email_address_win_adviser_name_and_more.py @@ -0,0 +1,52 @@ +# Generated by Django 4.2.10 on 2024-05-02 08:03 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('company', '0141_auto_20240222_1534'), + ('export_win', '0038_alter_winadviser_options_win_migrated_on'), + ] + + operations = [ + migrations.AddField( + model_name='win', + name='adviser_email_address', + field=models.EmailField(blank=True, max_length=254, verbose_name='Adviser email address'), + ), + migrations.AddField( + model_name='win', + name='adviser_name', + field=models.CharField(blank=True, help_text='This is the name of the adviser who created the Win', max_length=128, verbose_name='Adviser name'), + ), + migrations.AlterField( + model_name='breakdown', + name='year', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='win', + name='adviser', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='wins', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='win', + name='company', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='wins', to='company.company'), + ), + migrations.AlterField( + model_name='win', + name='lead_officer', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='lead_officer_wins', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='winadviser', + name='adviser', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='win_advisers', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/datahub/export_win/models.py b/datahub/export_win/models.py index ef2afeb3e..49f3ab5b1 100644 --- a/datahub/export_win/models.py +++ b/datahub/export_win/models.py @@ -3,7 +3,7 @@ from django.conf import settings from django.db import models, transaction -from django.db.models import Max, Sum +from django.db.models import Max from django.db.models.signals import post_save from django.dispatch import receiver @@ -15,9 +15,10 @@ Contact, ExportExperience, ) -from datahub.core import constants, reversion +from datahub.core import reversion from datahub.core.models import BaseModel, BaseOrderedConstantModel from datahub.export_win.constants import EXPORT_WINS_LEGACY_ID_START_VALUE +from datahub.export_win.utils import calculate_totals_for_export_win from datahub.metadata.models import ( Country, Sector, @@ -194,8 +195,32 @@ class Win(BaseModel): """Information about a given Export win, submitted by an officer.""" id = models.UUIDField(primary_key=True, default=uuid.uuid4) - adviser = models.ForeignKey(Advisor, related_name='wins', on_delete=models.PROTECT) - company = models.ForeignKey(Company, related_name='wins', on_delete=models.PROTECT) + adviser = models.ForeignKey( + Advisor, + related_name='wins', + on_delete=models.PROTECT, + null=True, + blank=True, + ) + # Legacy field + adviser_name = models.CharField( + max_length=128, + verbose_name='Adviser name', + help_text='This is the name of the adviser who created the Win', + blank=True, + ) + # Legacy field + adviser_email_address = models.EmailField( + verbose_name='Adviser email address', + blank=True, + ) + company = models.ForeignKey( + Company, + related_name='wins', + on_delete=models.PROTECT, + null=True, + blank=True, + ) # customers company_contacts = models.ManyToManyField(Contact, blank=True, related_name='wins') @@ -313,6 +338,8 @@ class Win(BaseModel): Advisor, related_name='lead_officer_wins', on_delete=models.PROTECT, + null=True, + blank=True, ) team_members = models.ManyToManyField( Advisor, @@ -417,11 +444,18 @@ class Win(BaseModel): objects = BaseExportWinSoftDeleteManager() def __str__(self): - return (f'Export win {self.pk}: {self.adviser} <{self.adviser.email}> - ' - f'{self.created_on.strftime("%Y-%m-%d %H:%M:%S") if self.created_on else ""}') + if self.adviser: + return (f'Export win {self.pk}: {self.adviser} <{self.adviser.email}> - ' + f'{self.created_on.strftime("%Y-%m-%d %H:%M:%S") if self.created_on else ""}') + else: + return ( + f'Export win {self.pk} (legacy): {self.adviser_name} ' + f'<{self.adviser_email_address}> - ' + f'{self.created_on.strftime("%Y-%m-%d %H:%M:%S") if self.created_on else ""}' + ) def save(self, *args, **kwargs): - calc_total = _calculate_totals_for_export_win(self) + calc_total = calculate_totals_for_export_win(self) self.total_expected_export_value = calc_total['total_export_value'] self.total_expected_non_export_value = calc_total['total_non_export_value'] self.total_expected_odi_value = calc_total['total_odi_value'] @@ -437,7 +471,7 @@ class Breakdown(BaseModel, BaseLegacyModel): related_name='breakdowns', on_delete=models.PROTECT, ) - year = models.PositiveIntegerField() + year = models.IntegerField() value = models.BigIntegerField() @@ -448,6 +482,8 @@ class WinAdviser(BaseModel, BaseLegacyModel): Advisor, related_name='win_advisers', on_delete=models.PROTECT, + null=True, + blank=True, ) win = models.ForeignKey(Win, related_name='advisers', on_delete=models.CASCADE) team_type = models.ForeignKey( @@ -681,33 +717,12 @@ class Meta: proxy = True -def _calculate_totals_for_export_win(win_instance): - """Base class for Total Export, Non Export and ODI""" - export_type_value = constants.BreakdownType.export.value - non_export_value = constants.BreakdownType.non_export.value - odi_value = constants.BreakdownType.odi.value - return { - 'total_export_value': win_instance.breakdowns.filter( - type_id=export_type_value.id).aggregate( - total_export_value=Sum('value'), - )['total_export_value'] or 0, - 'total_non_export_value': win_instance.breakdowns.filter( - type_id=non_export_value.id).aggregate( - total_non_export_value=Sum('value'), - )['total_non_export_value'] or 0, - 'total_odi_value': win_instance.breakdowns.filter( - type_id=odi_value.id).aggregate( - total_odi_value=Sum('value'), - )['total_odi_value'] or 0, - } - - @receiver(post_save, sender=Breakdown) def update_total_values(sender, instance, **kwargs): """Save the right total values""" win = instance.win - calc_total = _calculate_totals_for_export_win(win) + calc_total = calculate_totals_for_export_win(win) win.total_expected_export_value = calc_total['total_export_value'] win.total_expected_non_export_value = calc_total['total_non_export_value'] win.total_expected_odi_value = calc_total['total_odi_value'] diff --git a/datahub/export_win/test/test_admin.py b/datahub/export_win/test/test_admin.py index 8db45dcdb..dca401572 100644 --- a/datahub/export_win/test/test_admin.py +++ b/datahub/export_win/test/test_admin.py @@ -194,12 +194,20 @@ def test_get_queryset(): @pytest.mark.django_db -def test_get_adviser_name(): +def test_get_computed_adviser_name_adviser(): """Test for get adviser name""" adviser = AdviserFactory(first_name='John', last_name='Smith') win_adviser = WinAdviserFactory(adviser=adviser) admin_instance = WinAdviserAdmin(model=WinAdviser, admin_site=AdminSite()) - assert admin_instance.get_adviser_name(win_adviser) == 'John Smith' + assert admin_instance.get_computed_adviser_name(win_adviser) == 'John Smith' + + +@pytest.mark.django_db +def test_get_computed_adviser_name_legacy_adviser(): + """Test for get legacy adviser name""" + win_adviser = WinAdviserFactory(adviser=None, name='John Smith') + admin_instance = WinAdviserAdmin(model=WinAdviser, admin_site=AdminSite()) + assert admin_instance.get_computed_adviser_name(win_adviser) == 'John Smith' @pytest.mark.django_db diff --git a/datahub/export_win/test/test_export_wins_api.py b/datahub/export_win/test/test_export_wins_api.py new file mode 100644 index 000000000..31402cac2 --- /dev/null +++ b/datahub/export_win/test/test_export_wins_api.py @@ -0,0 +1,124 @@ +import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.test.utils import override_settings +from requests.exceptions import ( + ConnectionError, + ConnectTimeout, + ReadTimeout, + Timeout, +) +from rest_framework import status + +from datahub.company.export_wins_api import ( + ExportWinsAPIConnectionError, + ExportWinsAPIHTTPError, + ExportWinsAPITimeoutError, +) +from datahub.core.test_utils import APITestMixin, HawkMockJSONResponse +from datahub.export_win.export_wins_api import ( + get_legacy_export_wins_dataset, +) + + +class TestExportWinsDatasetApi(APITestMixin): + """ + Tests functionality to obtain export wins datasets. + """ + + @override_settings(EXPORT_WINS_SERVICE_BASE_URL=None) + def test_export_wins_api_missing_settings_error(self): + """ + Test when environment variables are not set an exception is thrown. + """ + with pytest.raises(ImproperlyConfigured): + next(get_legacy_export_wins_dataset('/data-hub-wins')) + + @pytest.mark.parametrize( + 'request_exception,expected_exception', + ( + ( + ConnectionError, + ExportWinsAPIConnectionError, + ), + ( + ConnectTimeout, + ExportWinsAPIConnectionError, + ), + ( + Timeout, + ExportWinsAPITimeoutError, + ), + ( + ReadTimeout, + ExportWinsAPITimeoutError, + ), + ), + ) + def test_export_wins_api_request_error( + self, + requests_mock, + request_exception, + expected_exception, + ): + """ + Test if there is an error connecting to export wins API + the expected exception was thrown. + """ + requests_mock.get( + '/data-hub-wins', + exc=request_exception, + ) + with pytest.raises(expected_exception): + next(get_legacy_export_wins_dataset('/data-hub-wins')) + + @pytest.mark.parametrize( + 'response_status', + ( + status.HTTP_400_BAD_REQUEST, + status.HTTP_401_UNAUTHORIZED, + status.HTTP_403_FORBIDDEN, + status.HTTP_404_NOT_FOUND, + status.HTTP_405_METHOD_NOT_ALLOWED, + status.HTTP_500_INTERNAL_SERVER_ERROR, + ), + ) + def test_export_wins_api_error( + self, + requests_mock, + response_status, + ): + """Test if the export wins api returns a status code that is not 200.""" + requests_mock.get( + '/data-hub-wins', + status_code=response_status, + ) + with pytest.raises( + ExportWinsAPIHTTPError, + match=f'The Export Wins API returned an error status: {response_status}', + ): + next(get_legacy_export_wins_dataset('/data-hub-wins')) + + def test_get_call_invoked( + self, + requests_mock, + ): + """ + Check GET call will be made + """ + dynamic_response = HawkMockJSONResponse( + api_id=settings.EXPORT_WINS_HAWK_ID, + api_key=settings.EXPORT_WINS_HAWK_KEY, + response={ + 'next': None, + 'results': [], + }, + ) + matcher = requests_mock.get( + '/data-hub-wins', + status_code=status.HTTP_200_OK, + text=dynamic_response, + ) + next(get_legacy_export_wins_dataset('/data-hub-wins')) + + assert matcher.called_once diff --git a/datahub/export_win/test/test_legacy_migration.py b/datahub/export_win/test/test_legacy_migration.py new file mode 100644 index 000000000..9228c7977 --- /dev/null +++ b/datahub/export_win/test/test_legacy_migration.py @@ -0,0 +1,390 @@ +from datetime import datetime, timezone + +import pytest + +from dateutil import parser + +from django.conf import settings +from freezegun import freeze_time + +from rest_framework import status + +from datahub.company.test.factories import ( + AdviserFactory, + CompanyFactory, + ContactFactory, +) +from datahub.core.test_utils import HawkMockJSONResponse +from datahub.export_win.legacy_migration import ( + migrate_all_legacy_wins, +) +from datahub.export_win.models import ( + Win, +) +from datahub.export_win.test.factories import ( + LegacyExportWinsToDataHubCompanyFactory, +) + +pytestmark = pytest.mark.django_db + + +mock_legacy_wins_page_urls = { + 'wins': [ + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-wins', + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-wins?cursor=1&source=L', + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-wins?cursor=2&source=E', + ], + 'breakdowns': [ + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-breakdowns', + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-breakdowns?cursor=1&source=L', + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-breakdowns?cursor=2&source=E', + ], + 'advisers': [ + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-advisers', + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-advisers?cursor=1&source=L', + f'{settings.EXPORT_WINS_SERVICE_BASE_URL}/data-hub-advisers?cursor=2&source=E', + ], +} + +legacy_wins = { + mock_legacy_wins_page_urls['wins'][0]: { + 'next': mock_legacy_wins_page_urls['wins'][1], + 'results': [{ + 'associated_programme_1': 1, + 'associated_programme_2': 2, + 'associated_programme_3': 3, + 'associated_programme_4': 9, + 'associated_programme_5': 11, + 'audit': None, + 'business_potential': None, + 'business_type': '4', + 'cdms_reference': 'abcd', + 'company_name': 'Lambda', + 'complete': True, + 'confirmation__access_to_contacts': 5, + 'confirmation__access_to_information': 5, + 'confirmation__agree_with_win': None, + 'confirmation__case_study_willing': False, + 'confirmation__comments': '', + 'confirmation__company_was_at_risk_of_not_exporting': True, + 'confirmation__created': '2024-01-23T01:12:47.221146Z', + 'confirmation__developed_relationships': 1, + 'confirmation__gained_confidence': 3, + 'confirmation__has_enabled_expansion_into_existing_market': False, + 'confirmation__has_enabled_expansion_into_new_market': True, + 'confirmation__has_explicit_export_plans': False, + 'confirmation__has_increased_exports_as_percent_of_turnover': True, + 'confirmation__improved_profile': 3, + 'confirmation__interventions_were_prerequisite': False, + 'confirmation__involved_state_enterprise': False, + 'confirmation__name': 'John M Doe', + 'confirmation__other_marketing_source': 'Web', + 'confirmation__our_support': 0, + 'confirmation__overcame_problem': 0, + 'confirmation__support_improved_speed': True, + 'country': 'AT', + 'created': '2024-01-22T01:12:47.221126Z', + 'customer_email_address': 'test@test.email', + 'customer_job_title': 'Director', + 'customer_location': 1, + 'customer_name': 'John M Doe', + 'date': '2024-02-01', + 'description': 'Lorem ipsum', + 'export_experience': None, + 'goods_vs_services': 1, + 'has_hvo_specialist_involvement': False, + 'hq_team': 'itt:DIT Team East Midlands - International Trade Team', + 'hvc': None, + 'hvo_programme': '', + 'id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + 'is_e_exported': False, + 'is_line_manager_confirmed': True, + 'is_personally_confirmed': True, + 'is_prosperity_fund_related': False, + 'lead_officer_email_address': '', + 'lead_officer_name': 'Jane Doe', + 'line_manager_name': 'Linem Anager', + 'name_of_customer': '', + 'name_of_export': '', + 'other_official_email_address': '', + 'team_type': 'itt', + 'type_of_support_1': 1, + 'type_of_support_2': 2, + 'type_of_support_3': None, + 'user__email': 'user.email@trade.gov.uk', + 'user__name': 'User Email', + 'sector_display': 'Creative industries : Art', + 'confirmation_last_export': + 'Apart from this win, we have exported in the last 12 months', + 'confirmation_marketing_source': 'Don’t know', + 'confirmation_portion_without_help': 'No value without our help', + 'country_name': 'Austria', + }], + }, + mock_legacy_wins_page_urls['wins'][1]: { + 'next': mock_legacy_wins_page_urls['wins'][2], + 'results': [{ + 'associated_programme_1': 1, + 'associated_programme_2': 2, + 'associated_programme_3': None, + 'associated_programme_4': None, + 'associated_programme_5': None, + 'audit': None, + 'business_potential': None, + 'business_type': '3', + 'cdms_reference': 'abcd', + 'company_name': 'Lambda', + 'complete': True, + 'confirmation__access_to_contacts': 5, + 'confirmation__access_to_information': 5, + 'confirmation__agree_with_win': None, + 'confirmation__case_study_willing': False, + 'confirmation__comments': '', + 'confirmation__company_was_at_risk_of_not_exporting': True, + 'confirmation__created': '2024-02-25T01:12:48.111131Z', + 'confirmation__developed_relationships': 1, + 'confirmation__gained_confidence': 3, + 'confirmation__has_enabled_expansion_into_existing_market': False, + 'confirmation__has_enabled_expansion_into_new_market': True, + 'confirmation__has_explicit_export_plans': False, + 'confirmation__has_increased_exports_as_percent_of_turnover': True, + 'confirmation__improved_profile': 3, + 'confirmation__interventions_were_prerequisite': False, + 'confirmation__involved_state_enterprise': False, + 'confirmation__name': 'John Doe', + 'confirmation__other_marketing_source': 'Web', + 'confirmation__our_support': 0, + 'confirmation__overcame_problem': 0, + 'confirmation__support_improved_speed': True, + 'country': 'US', + 'created': '2024-02-24T01:12:47.221126Z', + 'customer_email_address': 'test@test.email', + 'customer_job_title': 'Director', + 'customer_location': 1, + 'customer_name': 'John Doe', + 'date': '2024-04-01', + 'description': 'Lorem ipsum', + 'export_experience': None, + 'goods_vs_services': 1, + 'has_hvo_specialist_involvement': False, + 'hq_team': 'itt:DIT Team East Midlands - International Trade Team', + 'hvc': None, + 'hvo_programme': '', + 'id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + 'is_e_exported': False, + 'is_line_manager_confirmed': True, + 'is_personally_confirmed': True, + 'is_prosperity_fund_related': False, + 'lead_officer_email_address': '', + 'lead_officer_name': 'Jane Smith', + 'line_manager_name': 'Linem Anager', + 'name_of_customer': '', + 'name_of_export': '', + 'other_official_email_address': '', + 'team_type': 'itt', + 'type_of_support_1': 1, + 'type_of_support_2': 2, + 'type_of_support_3': None, + 'user__email': 'user.email1@trade.gov.uk', + 'user__name': 'User Email', + 'sector_display': 'Creative industries : Art', + 'confirmation_last_export': + 'Apart from this win, we have exported in the last 12 months', + 'confirmation_marketing_source': 'Don’t know', + 'confirmation_portion_without_help': 'No value without our help', + 'country_name': 'Austria', + }], + }, + mock_legacy_wins_page_urls['breakdowns'][0]: { + 'next': mock_legacy_wins_page_urls['breakdowns'][1], + 'results': [ + { + 'id': 5, + 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + 'type': 1, + 'year': 2023, + 'value': 3000, + }, + { + 'id': 6, + 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + 'type': 1, + 'year': 2024, + 'value': 4000, + }, + { + 'id': 7, + 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + 'type': 1, + 'year': 2025, + 'value': 5000, + }, + { + 'id': 8, + 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + 'type': 1, + 'year': 2026, + 'value': 6000, + }, + { + 'id': 9, + 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + 'type': 1, + 'year': 2027, + 'value': 3000, + }, + ], + }, + mock_legacy_wins_page_urls['breakdowns'][1]: { + 'next': mock_legacy_wins_page_urls['breakdowns'][2], + 'results': [ + { + 'id': 10, + 'win__id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + 'type': 1, + 'year': 2024, + 'value': 3000, + }, + { + 'id': 11, + 'win__id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + 'type': 1, + 'year': 2025, + 'value': 4000, + }, + { + 'id': 12, + 'win__id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + 'type': 1, + 'year': 2026, + 'value': 5000, + }, + { + 'id': 13, + 'win__id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + 'type': 1, + 'year': 2027, + 'value': 6000, + }, + { + 'id': 14, + 'win__id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + 'type': 1, + 'year': 2028, + 'value': 3000, + }, + ], + }, + mock_legacy_wins_page_urls['advisers'][0]: { + 'next': mock_legacy_wins_page_urls['advisers'][1], + 'results': [ + { + 'hq_team': 'itt:The North West International Trade Team', + 'id': 1, + 'location': 'Manchester', + 'name': 'John Doe', + 'team_type': 'itt', + 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c', + }, + ], + }, + mock_legacy_wins_page_urls['advisers'][1]: { + 'next': mock_legacy_wins_page_urls['advisers'][2], + 'results': [ + { + 'hq_team': 'itt:The North West International Trade Team', + 'id': 1, + 'location': 'London', + 'name': 'John Smith', + 'team_type': 'itt', + 'win__id': '02ce5d82-5294-477a-ab9a-94782e7b2794', + }, + ], + }, +} + + +@pytest.fixture +def mock_legacy_wins_pages(requests_mock): + for url, data in legacy_wins.items(): + dynamic_response = HawkMockJSONResponse( + api_id=settings.EXPORT_WINS_HAWK_ID, + api_key=settings.EXPORT_WINS_HAWK_KEY, + response=data, + ) + requests_mock.get( + url.replace(settings.EXPORT_WINS_SERVICE_BASE_URL, ''), + status_code=status.HTTP_200_OK, + text=dynamic_response, + ) + + +def test_legacy_migration(mock_legacy_wins_pages): + """Tests that legacy wins are migrated.""" + company = CompanyFactory() + LegacyExportWinsToDataHubCompanyFactory( + id='4c90a214-035f-4445-b6a1-ca7af3486f8c', + company=company, + ) + jane_doe = AdviserFactory( + first_name='Jane', + last_name='Doe', + ) + john_smith = AdviserFactory( + first_name='John', + last_name='Smith', + ) + adviser = AdviserFactory( + contact_email='user.email@trade.gov.uk', + ) + contact = ContactFactory( + company=company, + first_name='John', + last_name='Doe', + ) + + current_date = datetime.now(tz=timezone.utc) + + with freeze_time(current_date): + migrate_all_legacy_wins() + + win_1 = Win.objects.get(id='4c90a214-035f-4445-b6a1-ca7af3486f8c') + assert win_1.company_contacts.first() == contact + assert win_1.company == company + assert win_1.lead_officer == jane_doe + assert win_1.adviser == adviser + assert win_1.total_expected_export_value == 21000 + assert win_1.total_expected_non_export_value == 0 + assert win_1.total_expected_odi_value == 0 + assert win_1.breakdowns.count() == 5 + assert win_1.migrated_on == current_date + win_1_created_on = parser.parse('2024-01-22T01:12:47.221126Z').astimezone(timezone.utc) + assert win_1.created_on == win_1_created_on + assert win_1.customer_response.created_on == win_1_created_on + win_1_responded_on = parser.parse('2024-01-23T01:12:47.221146Z').astimezone(timezone.utc) + assert win_1.customer_response.responded_on == win_1_responded_on + + win_1_adviser = win_1.advisers.first() + assert win_1_adviser.adviser is None + assert win_1_adviser.name == 'John Doe' + + win_2 = Win.objects.get(id='02ce5d82-5294-477a-ab9a-94782e7b2794') + assert win_2.company_contacts.count() == 0 + assert win_2.company is None + assert win_2.company_name == 'Lambda' + assert win_2.customer_name == 'John Doe' + assert win_2.lead_officer is None + assert win_2.lead_officer_name == 'Jane Smith' + assert win_2.adviser_email_address == 'user.email1@trade.gov.uk' + assert win_2.adviser is None + assert win_2.total_expected_export_value == 21000 + assert win_2.total_expected_non_export_value == 0 + assert win_2.total_expected_odi_value == 0 + assert win_2.breakdowns.count() == 5 + assert win_2.advisers.first().adviser == john_smith + assert win_2.migrated_on == current_date + win_2_created_on = parser.parse('2024-02-24T01:12:47.221126Z').astimezone(timezone.utc) + assert win_2.created_on == win_2_created_on + assert win_2.customer_response.created_on == win_2_created_on + win_2_responded_on = parser.parse('2024-02-25T01:12:48.111131Z').astimezone(timezone.utc) + assert win_2.customer_response.responded_on == win_2_responded_on diff --git a/datahub/export_win/test/test_models.py b/datahub/export_win/test/test_models.py index c649dcc13..1d7c96963 100644 --- a/datahub/export_win/test/test_models.py +++ b/datahub/export_win/test/test_models.py @@ -3,11 +3,11 @@ from datahub.company.test.factories import AdviserFactory from datahub.export_win.constants import EXPORT_WINS_LEGACY_ID_START_VALUE from datahub.export_win.models import ( - _calculate_totals_for_export_win, Breakdown, update_total_values, WinAdviser) from datahub.export_win.test.factories import BreakdownFactory, WinAdviserFactory, WinFactory +from datahub.export_win.utils import calculate_totals_for_export_win pytestmark = pytest.mark.django_db @@ -106,7 +106,7 @@ class TestWinModel(): def test_win_save(self, win_factory): win = win_factory - calc_total = _calculate_totals_for_export_win(win) + calc_total = calculate_totals_for_export_win(win) win.save() assert win.total_expected_export_value == calc_total['total_export_value'] assert win.total_expected_non_export_value == calc_total['total_non_export_value'] @@ -115,7 +115,7 @@ def test_win_save(self, win_factory): def test_update_total_values(self, adviser_factory, win_factory, breakdown_factory): win = win_factory breakdown = breakdown_factory - calc_total = _calculate_totals_for_export_win(win) + calc_total = calculate_totals_for_export_win(win) expected_export_value = calc_total['total_export_value'] expected_non_export_value = calc_total['total_non_export_value'] expected_odi_value = calc_total['total_odi_value'] diff --git a/datahub/export_win/utils.py b/datahub/export_win/utils.py new file mode 100644 index 000000000..ae63ef6bf --- /dev/null +++ b/datahub/export_win/utils.py @@ -0,0 +1,24 @@ +from django.db.models import Sum + +from datahub.core import constants + + +def calculate_totals_for_export_win(win_instance): + """Base class for Total Export, Non Export and ODI""" + export_type_value = constants.BreakdownType.export.value + non_export_value = constants.BreakdownType.non_export.value + odi_value = constants.BreakdownType.odi.value + return { + 'total_export_value': win_instance.breakdowns.filter( + type_id=export_type_value.id).aggregate( + total_export_value=Sum('value'), + )['total_export_value'] or 0, + 'total_non_export_value': win_instance.breakdowns.filter( + type_id=non_export_value.id).aggregate( + total_non_export_value=Sum('value'), + )['total_non_export_value'] or 0, + 'total_odi_value': win_instance.breakdowns.filter( + type_id=odi_value.id).aggregate( + total_odi_value=Sum('value'), + )['total_odi_value'] or 0, + }