From 65029a75f1bcab029f3f9c8a1fdc3991c21ce047 Mon Sep 17 00:00:00 2001 From: Cameron Lamb Date: Thu, 28 Sep 2023 17:34:03 +0100 Subject: [PATCH 1/4] Update PII settings --- pii-secret-exclude.txt | 34 +++++++++---------- pyproject.toml | 2 +- scripts/truncate_anonymise.py.cli | 2 +- src/e2e_tests/conftest.py | 6 ++-- src/feedback/test/test_utils.py | 16 +++++---- src/peoplefinder/backports.py | 2 +- .../legacy_migration/migrate_legacy.py | 4 +-- ..._alter_person_fluent_languages_and_more.py | 2 +- src/peoplefinder/models.py | 2 +- src/peoplefinder/services/person.py | 2 +- .../test/services/test_profile.py | 4 +-- .../test/views/test_delete_profile_views.py | 6 ++-- .../test/views/test_profile_views.py | 6 ++-- src/peoplefinder/test/views/test_search.py | 2 +- src/peoplefinder/test/views/test_team.py | 8 ++--- 15 files changed, 51 insertions(+), 47 deletions(-) diff --git a/pii-secret-exclude.txt b/pii-secret-exclude.txt index 43f1e5cfd..94267ec47 100644 --- a/pii-secret-exclude.txt +++ b/pii-secret-exclude.txt @@ -1,25 +1,25 @@ .env.ci .env.example .pre-commit-config.yaml -Procfile -README.md -config/stop-words.txt -config/synonyms.txt -core/templates/404.html -core/test/test_views.py +docs/features/search.excalidraw import_wordpress/test/wordpress_test.xml +Makefile package-lock.json -peoplefinder/test/conftest.py -peoplefinder/test/services/test_audit_log.py +poetry.lock +Procfile +README.md requirements.txt -requirements/base.txt -requirements/dev.txt -requirements/prod.txt runtime.txt setup/init.sql -user/management/commands/create_test_users.py -user/migrations/0001_initial.py -user/migrations/0002_auto_20210216_0650.py -user/migrations/0004_historicaluser.py -user/test/factories.py -user/test/test_user.py +src/config/stop-words.txt +src/config/synonyms.txt +src/core/templates/404.html +src/core/test/test_views.py +src/peoplefinder/test/conftest.py +src/peoplefinder/test/services/test_audit_log.py +src/user/management/commands/create_test_users.py +src/user/migrations/0001_initial.py +src/user/migrations/0002_auto_20210216_0650.py +src/user/migrations/0004_historicaluser.py +src/user/test/factories.py +src/user/test/test_user.py diff --git a/pyproject.toml b/pyproject.toml index ec82236bc..9173642ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "digital-workspace-v2" version = "1.0.0" description = "" -authors = ["Live Services "] +authors = ["Live Services "] # /PS-IGNORE license = "MIT" readme = "README.md" # package-mode = false - Paketo buildpack breaks since it's at v1.7.1 diff --git a/scripts/truncate_anonymise.py.cli b/scripts/truncate_anonymise.py.cli index 9fb9f6b0d..523f669a1 100644 --- a/scripts/truncate_anonymise.py.cli +++ b/scripts/truncate_anonymise.py.cli @@ -64,7 +64,7 @@ for person in Person.objects.all().reverse(): f"{person.first_name.lower()}.{person.last_name.lower()}@{email_parts[1]}" ) person.user.username = ( - f"{person.first_name.lower()}.{person.last_name.lower()}-123456@id.trade.gov.uk" + f"{person.first_name.lower()}.{person.last_name.lower()}-123456@id.trade.gov.uk" # /PS-IGNORE ) person.user.email = person.email if person.user.sso_contact_email is not None: diff --git a/src/e2e_tests/conftest.py b/src/e2e_tests/conftest.py index bb390013f..3d3d47d56 100644 --- a/src/e2e_tests/conftest.py +++ b/src/e2e_tests/conftest.py @@ -120,7 +120,7 @@ def recreate_db_between_tests(django_db_blocker): @pytest.fixture def superuser(django_db_blocker, django_user_model, page): - email = "super.user@example.com" + email = "super.user@example.com" # /PS-IGNORE user, _ = django_user_model.objects.get_or_create( username="testsuperuser", @@ -144,10 +144,10 @@ def superuser(django_db_blocker, django_user_model, page): @pytest.fixture def user(django_db_blocker, django_user_model): user, _ = django_user_model.objects.get_or_create( - username="john.smith-1234abcd@digital.trade.gov.uk", + username="john.smith-1234abcd@digital.trade.gov.uk", # /PS-IGNORE first_name="John", last_name="Smith", - email="john.smith@digital.trade.gov.uk", + email="john.smith@digital.trade.gov.uk", # /PS-IGNORE legacy_sso_user_id="1234abcd-1234-abcd-1234-abcd1234abcd", ) user.set_password("password") diff --git a/src/feedback/test/test_utils.py b/src/feedback/test/test_utils.py index f63f44c71..0178137d7 100644 --- a/src/feedback/test/test_utils.py +++ b/src/feedback/test/test_utils.py @@ -1,5 +1,6 @@ -import pytest from unittest.mock import call + +import pytest from django.test import override_settings from django_feedback_govuk.models import BaseFeedback @@ -56,7 +57,7 @@ def test_send_feedback_notification_with_no_email_recipients(): @override_settings( GOVUK_NOTIFY_API_KEY="this-is-my-really-long-api-key-because-gov-uk-notify-expects-it-to-be-long-when-you-create-a-service", - FEEDBACK_NOTIFICATION_EMAIL_RECIPIENTS=["test@email.com"], + FEEDBACK_NOTIFICATION_EMAIL_RECIPIENTS=["test@email.com"], # /PS-IGNORE FEEDBACK_NOTIFICATION_EMAIL_TEMPLATE_ID="test-template-id", WAGTAILADMIN_BASE_URL="https://test.example.com/", ) @@ -67,7 +68,7 @@ def test_send_feedback_notification_with_valid_settings(mocker): ) send_feedback_notification() mock_send_email_notification.assert_called_once_with( - email_address="test@email.com", + email_address="test@email.com", # /PS-IGNORE template_id="test-template-id", personalisation={ "feedback_url": "https://test.example.com/feedback/submitted/" @@ -77,7 +78,10 @@ def test_send_feedback_notification_with_valid_settings(mocker): @override_settings( GOVUK_NOTIFY_API_KEY="this-is-my-really-long-api-key-because-gov-uk-notify-expects-it-to-be-long-when-you-create-a-service", - FEEDBACK_NOTIFICATION_EMAIL_RECIPIENTS=["test1@email.com", "test2@email.com"], + FEEDBACK_NOTIFICATION_EMAIL_RECIPIENTS=[ + "test1@email.com", # /PS-IGNORE + "test2@email.com", # /PS-IGNORE + ], FEEDBACK_NOTIFICATION_EMAIL_TEMPLATE_ID="test-template-id", WAGTAILADMIN_BASE_URL="https://test.example.com/", ) @@ -88,14 +92,14 @@ def test_send_feedback_notification_with_multiple_emails(mocker): ) expected_calls = [ call( - email_address="test1@email.com", + email_address="test1@email.com", # /PS-IGNORE template_id="test-template-id", personalisation={ "feedback_url": "https://test.example.com/feedback/submitted/" }, ), call( - email_address="test2@email.com", + email_address="test2@email.com", # /PS-IGNORE template_id="test-template-id", personalisation={ "feedback_url": "https://test.example.com/feedback/submitted/" diff --git a/src/peoplefinder/backports.py b/src/peoplefinder/backports.py index 818af6c7e..6b5436dbf 100644 --- a/src/peoplefinder/backports.py +++ b/src/peoplefinder/backports.py @@ -5,7 +5,7 @@ # Backport of Django's Postgres specific ArraySubquery expression. # Documentation: https://docs.djangoproject.com/en/dev/ref/contrib/postgres/expressions/#arraysubquery-expressions -# Commit: https://github.com/django/django/commit/a06b977a91f043c509df781670fb4cf35cb437b7 +# Commit: https://github.com/django/django/commit/a06b977a91f043c509df781670fb4cf35cb437b7 /PS-IGNORE # TODO: Remove once Django has released this and we upgrade to that version. class ArraySubquery(Subquery): template = "ARRAY(%(subquery)s)" diff --git a/src/peoplefinder/legacy_migration/migrate_legacy.py b/src/peoplefinder/legacy_migration/migrate_legacy.py index 57a2a1db1..160dc5231 100644 --- a/src/peoplefinder/legacy_migration/migrate_legacy.py +++ b/src/peoplefinder/legacy_migration/migrate_legacy.py @@ -242,11 +242,11 @@ def migrate_person(legacy_person, person): if legacy_person.ditsso_user_id: person.legacy_sso_user_id = legacy_person.ditsso_user_id - # first name + # first name /PS-IGNORE if legacy_person.given_name: person.first_name = legacy_person.given_name - # last name + # last name /PS-IGNORE if legacy_person.surname: person.last_name = legacy_person.surname diff --git a/src/peoplefinder/migrations/0115_alter_person_fluent_languages_and_more.py b/src/peoplefinder/migrations/0115_alter_person_fluent_languages_and_more.py index c3f8f1b57..2a207c7f1 100644 --- a/src/peoplefinder/migrations/0115_alter_person_fluent_languages_and_more.py +++ b/src/peoplefinder/migrations/0115_alter_person_fluent_languages_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): name="preferred_first_name", field=models.CharField( blank=True, - help_text="This name appears on your profile. Colleagues can search for you using either of your first names", + help_text="This name appears on your profile. Colleagues can search for you using either of your first names", # /PS-IGNORE max_length=200, null=True, ), diff --git a/src/peoplefinder/models.py b/src/peoplefinder/models.py index 1c703c63f..dec034c36 100644 --- a/src/peoplefinder/models.py +++ b/src/peoplefinder/models.py @@ -393,7 +393,7 @@ class RemoteWorking(models.TextChoices): max_length=200, help_text=( "This name appears on your profile. Colleagues can search for you" - " using either of your first names" + " using either of your first names" # /PS-IGNORE ), null=True, blank=True, diff --git a/src/peoplefinder/services/person.py b/src/peoplefinder/services/person.py index 7e4923f98..6f494baa8 100644 --- a/src/peoplefinder/services/person.py +++ b/src/peoplefinder/services/person.py @@ -197,7 +197,7 @@ def create_user_profile(self, user: User) -> Person: Q(legacy_sso_user_id=user.legacy_sso_user_id), # Next see if we can match on the email. Q(email=user.email), - # Finally try and match on the first and last name. + # Finally try and match on the first and last names /PS-IGNORE. Q(first_name=user.first_name, last_name=user.last_name), ] diff --git a/src/peoplefinder/test/services/test_profile.py b/src/peoplefinder/test/services/test_profile.py index 8942c7dab..67eb6b887 100644 --- a/src/peoplefinder/test/services/test_profile.py +++ b/src/peoplefinder/test/services/test_profile.py @@ -96,7 +96,7 @@ def test_already_has_profile(self, normal_user): { "user": None, "legacy_sso_user_id": "not-a-match", - "email": "not-a-match@example.com", + "email": "not-a-match@example.com", # /PS-IGNORE }, ), ) @@ -120,7 +120,7 @@ def test_no_match(self, normal_user): profile.user = None profile.legacy_sso_user_id = "not-a-match" - profile.email = "not-a-match@example.com" + profile.email = "not-a-match@example.com" # /PS-IGNORE profile.first_name = "no" profile.last_name = "match" profile.save() diff --git a/src/peoplefinder/test/views/test_delete_profile_views.py b/src/peoplefinder/test/views/test_delete_profile_views.py index 6b2d7f486..11fbb2209 100644 --- a/src/peoplefinder/test/views/test_delete_profile_views.py +++ b/src/peoplefinder/test/views/test_delete_profile_views.py @@ -179,10 +179,10 @@ def test_delete_profile_with_no_user(state): other_user = UserFactory( first_name="Other", last_name="User", - email="other.user@example.com", + email="other.user@example.com", # /PS-IGNORE legacy_sso_user_id=None, - username="other.user-11111111@example.com", - sso_contact_email="other.user@example.com", + username="other.user-11111111@example.com", # /PS-IGNORE + sso_contact_email="other.user@example.com", # /PS-IGNORE ) other_user.save() other_person = PersonService().create_user_profile(other_user) diff --git a/src/peoplefinder/test/views/test_profile_views.py b/src/peoplefinder/test/views/test_profile_views.py index 91fa5a2c2..9268dd867 100644 --- a/src/peoplefinder/test/views/test_profile_views.py +++ b/src/peoplefinder/test/views/test_profile_views.py @@ -424,13 +424,13 @@ def test_profile_edit_contact_view(state): response = state.client.get(view_url) assert response.status_code == 200 - assert state.person.contact_email == "jane.smith@test.com" + assert state.person.contact_email == "jane.smith@test.com" # /PS-IGNORE assert state.person.primary_phone_number is None assert state.person.secondary_phone_number is None form = ContactProfileEditForm( { - "contact_email": "jane.smith123@test.com", + "contact_email": "jane.smith123@test.com", # /PS-IGNORE "primary_phone_number": "01234567890", "secondary_phone_number": "09876543210", }, @@ -445,7 +445,7 @@ def test_profile_edit_contact_view(state): assert response.status_code == 302 assert response.url == view_url - assert state.person.contact_email == "jane.smith123@test.com" + assert state.person.contact_email == "jane.smith123@test.com" # /PS-IGNORE assert state.person.primary_phone_number == "01234567890" assert state.person.secondary_phone_number == "09876543210" diff --git a/src/peoplefinder/test/views/test_search.py b/src/peoplefinder/test/views/test_search.py index 2f94d4563..5e3aba844 100644 --- a/src/peoplefinder/test/views/test_search.py +++ b/src/peoplefinder/test/views/test_search.py @@ -43,7 +43,7 @@ def test_updated_profile(self, another_normal_user): another_normal_user.profile.first_name = "Tim" another_normal_user.profile.preferred_first_name = "Tim" - another_normal_user.profile.email = "tim.smith@example.com" + another_normal_user.profile.email = "tim.smith@example.com" # /PS-IGNORE another_normal_user.profile.save() call_command("update_index") diff --git a/src/peoplefinder/test/views/test_team.py b/src/peoplefinder/test/views/test_team.py index b527c0d16..3c9113026 100644 --- a/src/peoplefinder/test/views/test_team.py +++ b/src/peoplefinder/test/views/test_team.py @@ -21,18 +21,18 @@ def test_order_team_leaders(self, team_admin_user, software_team): red_leader = UserFactory( first_name="Red", last_name="Leader", - email="red.leader@example.com", + email="red.leader@example.com", # /PS-IGNORE legacy_sso_user_id=None, username="red-leader", - sso_contact_email="red.leader@example.com", + sso_contact_email="red.leader@example.com", # /PS-IGNORE ) gold_leader = UserFactory( first_name="Gold", last_name="Leader", - email="gold.leader@example.com", + email="gold.leader@example.com", # /PS-IGNORE legacy_sso_user_id=None, username="gold-leader", - sso_contact_email="gold.leader@example.com", + sso_contact_email="gold.leader@example.com", # /PS-IGNORE ) call_command("create_user_profiles") From 39c070151b0cdc99390c977ad971361292b493b2 Mon Sep 17 00:00:00 2001 From: Cameron Lamb Date: Thu, 5 Oct 2023 12:12:25 +0100 Subject: [PATCH 2/4] Update PII Why is `.filtered li:not([data-match` not working? --- pii-ner-exclude.txt | 67 +++++++++++++++++++ pii-secret-exclude.txt | 1 + src/core/templates/core/deactivated.html | 2 +- src/core/templates/includes/footer.html | 2 +- src/core/templates/includes/header.html | 78 +++++++++++------------ src/core/templates/wagtailadmin/base.html | 2 +- 6 files changed, 110 insertions(+), 42 deletions(-) diff --git a/pii-ner-exclude.txt b/pii-ner-exclude.txt index 871a7ea75..d5ac6fb0e 100644 --- a/pii-ner-exclude.txt +++ b/pii-ner-exclude.txt @@ -1,9 +1,11 @@ -O codecov.sh https://codecov.io/bash +.filtered li:not([data-match › " INSERT INTO peoplefinder_teamtree ( " INSERT INTO public.wagtaildocs_document " INSERT INTO public.wagtailimages_image " INSERT INTO public.wagtailmedia_media +"Annotate the queryset "Submit that the given person "Wordpress GUID { item.action|title } @@ -58,6 +60,7 @@ American Samoa Andorra Antarctica Antigua and Barbuda" +apply_active_classes=True %} AQ Args aria-controls="navigation @@ -70,12 +73,17 @@ ATOMIC_REBUILD AU audit log AuditLog +AUTHBROKER_CLIENT_ID AuthenticationFailed AuthenticationFailed +AuthenticationMiddleware +AutoField +AWS b"Add B006 b1b4b6 B901 +BACKEND backup_file_dev.backup backup.sql Bakery @@ -86,9 +94,11 @@ Bhutan body_no_html Botswana Bouvet Island +br>=0.4.5 django-environ>=0.4.5,<0.5 django-storages==1.11.1 +django.middleware.common DjangoCelery DjangoTemplates DK @@ -172,17 +194,26 @@ el = e.target Elastic ELASTIC_APM_SECRET_TOKEN ELLIPSIS +english EO errors.push('Photo EU +Europe event.preventDefault +F Y f"/people/{person.slug}/ f"/teams/{team.slug}/ f"Error F401 +F405 Falkland Islands Fast Track" FCO +FEEDBACK_FORMS +FEEDBACK_NOTIFICATION_EMAIL_TEMPLATE_ID +FIELD_COMMENT_LEGEND +FIELD_SATISFACTION_LEGEND +FileHandler FileReader found

network
.

href="https://www.nationalarchives.gov.uk +href="https://www.nationalarchives.gov.uk/information-management html { htmllaundry HTTP http://purl.org/dc/elements/1.1/ http://wordpress.org/export/1.2/ http://www.test.com/test/img-1.jpg +https https://docs.djangoproject.com/en/3.2/topics/http/sessions/#using-sessions-out-of-views https://pypi.org/project/django-settings-export/ https://stackoverflow.com/a/417184 @@ -248,6 +283,7 @@ https://www.googletagmanager.com/gtm.js?id='+i+dl+ ' HU IDE IGNORE_ANTI_VIRUS +img { img aria IMPLIED Import Wordpress @@ -258,7 +294,9 @@ Insert IntegerField International Trade is_superuser +item { item_id +j F Y Jan Mayen JE JM @@ -276,6 +314,7 @@ KN Lao People's Democratic Republic LearningInterest leaving{% +Level Navigation li { li class="quick-links li class="ws-whats @@ -295,9 +334,13 @@ manager_id markdown==3.3.4 Marshall Math.round((cropperData.x +max max_length=-1 +max-height +max-width McDonald Islands Mentor +MessageMiddleware method="GET Micronesia (Federated States Microsoft @@ -320,6 +363,7 @@ MZ NA NAME ner_output_file.txt +Network NetworksHome NEW_ASSET_PATH NewsPageNewsCategory @@ -339,6 +383,8 @@ Old Admiralty Building OPENSEARCH_JAVA_OPTS OPENSEARCH_JAVA_OPTS=-Xms512 OPTIONS +Order +page_tools_phrase_title_explicit Paraguay parent_id Parse Wordpress @@ -352,11 +398,13 @@ PERM_SEC_NAME person.roles.first PFM PFM_AWS_SECRET_ACCESS_KEY +Photo PHP PingdomConfig PM" PNG PoliciesHome +Polish Polynesia Privacy Policy PrivacyPolicy @@ -364,6 +412,7 @@ Private Office Profile Profile Editors" profile_completion__avg +PROFILE_DELETED_EMAIL_TEMPLATE_ID profile.contact_email ProfilePhoto Property / Estates @@ -379,21 +428,27 @@ RE re.match("documents_[0-9]+_title rect(1px RedirectMiddleware +RedisCache Relevanssi +Replicate request/ role="alert role="navigation roles__team__name +ROOT_LOG_LEVEL +rotate(90deg RS512 RU Russian S101 s3chunkuploader.fields +S3FileUploadHandler S703 SA Saint Helena @@ -412,6 +467,7 @@ self.seo_title SerializerMethodField Server-Authorization' SERVICE_NAME +SessionMiddleware Settings SG sha256 @@ -424,17 +480,24 @@ SN Software South Sudan SQL +SSO ST Stakeholder StreamHandler SV +svg { Syrian Arab Republic +taggit Tajikistan target="_blank Team +Team Admin Team Editors +Team.short_name team=1.26.5 +URLS +Use Markdown +UserConfig UUID v2 VA @@ -468,6 +534,7 @@ Video Video Card Video or Audio view_howdoipreview +vimeo VU W W503 diff --git a/pii-secret-exclude.txt b/pii-secret-exclude.txt index 94267ec47..e7c47b6e8 100644 --- a/pii-secret-exclude.txt +++ b/pii-secret-exclude.txt @@ -1,6 +1,7 @@ .env.ci .env.example .pre-commit-config.yaml +docs/contributing/testing.md docs/features/search.excalidraw import_wordpress/test/wordpress_test.xml Makefile diff --git a/src/core/templates/core/deactivated.html b/src/core/templates/core/deactivated.html index 608c9b55b..b070d0157 100644 --- a/src/core/templates/core/deactivated.html +++ b/src/core/templates/core/deactivated.html @@ -6,6 +6,6 @@

Your user ({{ user }}) has been deactivated

If this is incorrect, or you wish to be reactivated, please email - datahubsupport@uktrade.zendesk.com. + datahubsupport@uktrade.zendesk.com. {# /PS-IGNORE #}

{% endblock %} diff --git a/src/core/templates/includes/footer.html b/src/core/templates/includes/footer.html index b239b4655..cca9b2ca2 100644 --- a/src/core/templates/includes/footer.html +++ b/src/core/templates/includes/footer.html @@ -42,7 +42,7 @@

Support links

Feedback + href="mailto:digitalworkspace@digital.trade.gov.uk">Feedback {# /PS-IGNORE #} {% endif %} diff --git a/src/core/templates/includes/header.html b/src/core/templates/includes/header.html index f05910dee..94e59ef33 100644 --- a/src/core/templates/includes/header.html +++ b/src/core/templates/includes/header.html @@ -10,46 +10,46 @@ - Department for Business & Trade - - + src="{% webpack_static 'dbt-logo.png' %}" + srcset="{% webpack_static 'dbt-logo@2x.png' %} 2x" {# /PS-IGNORE #} + alt="Department for Business & Trade logo" /> + Department for Business & Trade + + + + + {% include 'includes/profile_panel.html' %} + - - {% include 'includes/profile_panel.html' %} - - - - - {% include 'includes/profile_panel.html' %} - + src="{% webpack_static 'dbt-logo.png' %}" + srcset="{% webpack_static 'dbt-logo@2x.png' %} 2x" + alt="Department for Business & Trade logo" /> + Department for Business & Trade + + + + {% include 'includes/profile_panel.html' %} + + -