Tracking pull request to merge release-2.5.0 to master (#2187)

* update to 2.5.0 * update PR 2187 * BC attorney address add and unit testing * js lint standard format * migration updated * TFRS-2126- Added instructional text * TFRS-2125- Added instructional text * Adding submissionDate tab and filter * TFRS-2158-Added org name to the email notification * TFRS-2182- Removed keycloak references * Removed commented lines * updating Lastupdate On filter * TFRS-2145-BC Attorney Address on Organization page * compliance report loading fix * fixing Insufficient available credit balance. Please adjust Line 26b error for compute_total endpoint * TFRS-2131- Added BC Attorney Address * simplified long variable names in the org address * Removed console.log and address_line_3 * feat: tfrs-2195 - changes to organization add/edit form * docker hub sunset (#2211) * updating filter for credit transaction * TFRS-2130- Prevent Supplier from updating address * Chore: Remove Remaining Keycloak References (#2214) * fix: removed remaining keycloak references and updated settings vars * fix: fixed tfrs logout ref * cleanup keycloak env vars * fix: tfrs-2195 - fixes/changes to organization add/edit form * add colon * fix: tfrs2145/2130 - fixes * more fixes * fix: penalty date fix * fixing pending credit transaction number display * fix: test updates, linting, snapshots * chore: org details cleanup * fix: tfrs-2158 - notification emails should include org name * fix: tfrs2131 - Display BC Attorney Address on Compliance and Exclusion Reports * fix: backend unit test updates and fixes * fix: filtering order fixes --------- Co-authored-by: jig-patel <[email protected]> Co-authored-by: Alex Zorkin <[email protected]> Co-authored-by: vibhishan <[email protected]> Co-authored-by: vibhiquartech <[email protected]> Co-authored-by: tim738745 <[email protected]> Co-authored-by: AlexZorkin <[email protected]>
bcgov · Apr 6, 2023 · a73fdb1 · a73fdb1
1 parent 25ea57f
commit a73fdb1
Show file tree

Hide file tree

Showing 63 changed files with 2,081 additions and 1,185 deletions.
diff --git a/.github/workflows/tfrs-release.yaml b/.github/workflows/tfrs-release.yaml
@@ -1,19 +1,22 @@
 ## For each release, the value of name, branches, RELEASE_NAME and PR_NUMBER need to be adjusted accordingly
 ## For each release, update lib/config.js: version and releaseBranch
 
-name: TFRS release-2.4.0
+name: TFRS release-2.5.0
 
 on:
   push:
-    branches: [ release-2.4.0 ]
+    branches: [ release-2.5.0 ]
+    paths:
+      - frontend/**
+      - backend/**
   workflow_dispatch:
   workflow_call:
 
 env:
   ## The pull request number of the Tracking pull request to merge the release branch to main
   ## Also remember to update the version in .pipeline/lib/config.js
-  PR_NUMBER: 2112
-  RELEASE_NAME: release-2.4.0
+  PR_NUMBER: 2187
+  RELEASE_NAME: release-2.5.0
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.pipeline/lib/config.js b/.pipeline/lib/config.js
@@ -1,7 +1,7 @@
 'use strict';
 const options= require('@bcgov/pipeline-cli').Util.parseArguments()
 const changeId = options.pr //aka pull-request
-const version = '2.4.0'
+const version = '2.5.0'
 const name = 'tfrs'
 const ocpName = 'apps.silver.devops'
 
@@ -13,7 +13,7 @@ options.git.repository='tfrs'
 const phases = {
   build: {  namespace:'0ab226-tools'    , name: `${name}`, phase: 'build'  , changeId:changeId, suffix: `-build-${changeId}`  , 
             instance: `${name}-build-${changeId}`  , version:`${version}-${changeId}`, tag:`build-${version}-${changeId}`,
-            releaseBranch: 'release-2.4.0'
+            releaseBranch: 'release-2.5.0'
           },
   dev: {namespace:'0ab226-dev'    , name: `${name}`, phase: 'dev'  , changeId:changeId, suffix: `-dev`  , 
         instance: `${name}-dev`  , version:`${version}`, tag:`dev-${version}`, dbServiceName: 'tfrs-spilo',
@@ -24,14 +24,7 @@ const phases = {
             frontendDebugEnabled: 'true',
         backendCpuRequest: '200m', backendCpuLimit: '400m', backendMemoryRequest: '600Mi', backendMemoryLimit: '1200Mi', backendHealthCheckDelay: 30, 
             backendHost: `tfrs-backend-dev.${ocpName}.gov.bc.ca`, backendReplicas: 2,
-            backendKeycloakSaBaseurl: 'https://dev.loginproxy.gov.bc.ca',
-            backendKeycloakSaClientId: 'tfrs-on-gold-4308',
-            backendKeycloakSaRealm: 'standard',
             backendKeycloakAudience: 'tfrs-on-gold-4308',
-            backendKeycloakCertsUrl: 'https://dev.loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/certs',
-            backendKeycloakClientId: 'tfrs-on-gold-4308',
-            backendKeycloakIssuer: 'https://dev.loginproxy.gov.bc.ca/auth/realms/standard',
-            backendKeycloakRealm: 'standard',
             backendWellKnownEndpoint: 'https://dev.loginproxy.gov.bc.ca/auth/realms/standard/.well-known/openid-configuration',
             celeryCpuRequest: '100m', celeryCpuLimit: '250m', celeryMemoryRequest: '1600Mi', celeryMemoryLimit: '3Gi',
         scanHandlerCpuRequest: '25m', scanHandlerCpuLimit: '50m', scanHandlerMemoryRequest: '50Mi', scanHandlerMemoryLimit: '100Mi',
@@ -53,14 +46,7 @@ const phases = {
             frontendDebugEnabled: 'true',
         backendCpuRequest: '200m', backendCpuLimit: '400m', backendMemoryRequest: '600Mi', backendMemoryLimit: '1200Mi', backendHealthCheckDelay: 30, 
             backendHost: `tfrs-backend-test.${ocpName}.gov.bc.ca`, backendReplicas: 4,
-            backendKeycloakSaBaseurl: 'https://test.loginproxy.gov.bc.ca',
-            backendKeycloakSaClientId: 'tfrs-on-gold-4308',
-            backendKeycloakSaRealm: 'standard',
             backendKeycloakAudience: 'tfrs-on-gold-4308',
-            backendKeycloakCertsUrl: 'https://test.loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/certs',
-            backendKeycloakClientId: 'tfrs-on-gold-4308',
-            backendKeycloakIssuer: 'https://test.loginproxy.gov.bc.ca/auth/realms/standard',
-            backendKeycloakRealm: 'standard',
             backendWellKnownEndpoint: 'https://test.loginproxy.gov.bc.ca/auth/realms/standard/.well-known/openid-configuration',
         celeryCpuRequest: '100m', celeryCpuLimit: '250m', celeryMemoryRequest: '1600Mi', celeryMemoryLimit: '3Gi',
         scanHandlerCpuRequest: '25m', scanHandlerCpuLimit: '50m', scanHandlerMemoryRequest: '50Mi', scanHandlerMemoryLimit: '100Mi',
@@ -82,14 +68,7 @@ const phases = {
             frontendDebugEnabled: 'false',
         backendCpuRequest: '200m', backendCpuLimit: '400m', backendMemoryRequest: '600Mi', backendMemoryLimit: '1200Mi', backendHealthCheckDelay: 30, 
             backendHost: `tfrs-backend-prod.${ocpName}.gov.bc.ca`, backendReplicas: 4,
-            backendKeycloakSaBaseurl: 'https://oidc.gov.bc.ca',
-            backendKeycloakSaClientId: 'tfrs-on-gold-4308',
-            backendKeycloakSaRealm: 'standard',
             backendKeycloakAudience: 'tfrs-on-gold-4308',
-            backendKeycloakCertsUrl: 'https://loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/certs',
-            backendKeycloakClientId: 'tfrs-on-gold-4308',
-            backendKeycloakIssuer: 'https://loginproxy.gov.bc.ca/auth/realms/standard',
-            backendKeycloakRealm: 'standard',
             backendWellKnownEndpoint: 'https://loginproxy.gov.bc.ca/auth/realms/standard/.well-known/openid-configuration',
         celeryCpuRequest: '100m', celeryCpuLimit: '250mm', celeryMemoryRequest: '1600Mi', celeryMemoryLimit: '3Gi',
         scanHandlerCpuRequest: '25m', scanHandlerCpuLimit: '50m', scanHandlerMemoryRequest: '50Mi', scanHandlerMemoryLimit: '100Mi',

diff --git a/.pipeline/lib/deploy.js b/.pipeline/lib/deploy.js
@@ -23,14 +23,7 @@ module.exports = settings => {
       'ENV_NAME': phases[phase].phase,
       'NAMESPACE': phases[phase].namespace,
       'VERSION': phases[phase].tag,
-      'KEYCLOAK_SA_BASEURL': phases[phase].backendKeycloakSaBaseurl,
-      'KEYCLOAK_SA_CLIENT_ID': phases[phase].backendKeycloakSaClientId,
-      'KEYCLOAK_SA_REALM': phases[phase].backendKeycloakSaRealm,
       'KEYCLOAK_AUDIENCE': phases[phase].backendKeycloakAudience,
-      'KEYCLOAK_CERTS_URL': phases[phase].backendKeycloakCertsUrl,
-      'KEYCLOAK_CLIENT_ID': phases[phase].backendKeycloakClientId,
-      'KEYCLOAK_ISSUER': phases[phase].backendKeycloakIssuer,
-      'KEYCLOAK_REALM':phases[phase].backendKeycloakRealm,
       'CPU_REQUEST':phases[phase].backendCpuRequest,
       'CPU_LIMIT':phases[phase].backendCpuLimit,
       'MEMORY_REQUEST':phases[phase].backendMemoryRequest,

diff --git a/backend/api/app.py b/backend/api/app.py
@@ -31,11 +31,10 @@
 from django.db.models.signals import post_migrate
 from minio import Minio
 
-from api.services.KeycloakAPI import list_users, get_token
 from db_comments.db_actions import create_db_comments, \
     create_db_comments_from_models
 from tfrs.settings import AMQP_CONNECTION_PARAMETERS, MINIO, DOCUMENTS_API, \
-    KEYCLOAK, EMAIL, TESTING, RUNSERVER
+    EMAIL, RUNSERVER
 
 
 class APIAppConfig(AppConfig):

diff --git a/backend/api/keycloak_authentication.py b/backend/api/keycloak_authentication.py
@@ -1,20 +1,18 @@
-import os
 import json
 import jwt
 import requests
 
 from django.core.cache import caches
 from django.conf import settings
 from django.db.models import Q
-from django.http import HttpResponseServerError
 from rest_framework import authentication
 from rest_framework import exceptions
 
 from api.models.User import User
 from api.models.UserCreationRequest import UserCreationRequest
 from api.models.UserLoginHistory import UserLoginHistory
-from api.services.KeycloakAPI import map_user
-from tfrs.settings import WELL_KNOWN_ENDPOINT
+from tfrs.settings import WELL_KNOWN_ENDPOINT, KEYCLOAK_AUDIENCE
+import tfrs.settings
 
 cache = caches['keycloak']
 
@@ -32,7 +30,8 @@ def refresh_jwk(self):
         self.jwks = jwks
 
     def __init__(self):
-        if not settings.KEYCLOAK['TESTING_ENABLED']:
+        self.unit_testing_enabled = getattr(tfrs.settings, 'UNIT_TESTING_ENABLED', False)
+        if not self.unit_testing_enabled:
             self.refresh_jwk()
 
     def create_login_history(self, user_token, success = False, error = None, path = ''):
@@ -58,9 +57,6 @@ def create_login_history(self, user_token, success = False, error = None, path =
 
     def authenticate(self, request):
         """Verify the JWT token and find the correct user in the DB"""
-        if not settings.KEYCLOAK['ENABLED']:
-            # fall through
-            return None
 
         auth = request.META.get('HTTP_AUTHORIZATION', None)
 
@@ -69,14 +65,15 @@ def authenticate(self, request):
             raise exceptions.AuthenticationFailed(
                 'Authorization header required')
 
-        if settings.KEYCLOAK['TESTING_ENABLED']:
+        if self.unit_testing_enabled:
             try:
                 user = User.objects.get(keycloak_user_id=auth['preferred_username'])
                 return user, None
             except User.DoesNotExist as exc:
                 print("Testing User does not exist")
                 raise User.DoesNotExist(str(exc))
 
+        print("auth", auth)
         try:
             scheme, token = auth.split()
         except ValueError:
@@ -112,7 +109,7 @@ def authenticate(self, request):
                 token,
                 signing_key.key,
                 algorithms=["RS256"],
-                audience=settings.KEYCLOAK['AUDIENCE'],
+                audience=KEYCLOAK_AUDIENCE,
                 options={"verify_exp": True},
             )
         except (jwt.InvalidTokenError, jwt.ExpiredSignature, jwt.DecodeError) as exc:

diff --git a/backend/api/migrations/0204_auto_20230321_0204.py b/backend/api/migrations/0204_auto_20230321_0204.py
@@ -0,0 +1,19 @@
+# Generated by Django 3.2.18 on 2023-03-21 02:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0203_auto_20230312_0823'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            "UPDATE organization_address SET address_line_2 = address_line_2 || ' ' || address_line_3;"
+        ),
+        migrations.RunSQL(
+            "UPDATE organization_address SET state = state || ' ' || county"
+        ),
+    ]
diff --git a/backend/api/migrations/0205_auto_20230321_0206.py b/backend/api/migrations/0205_auto_20230321_0206.py
@@ -0,0 +1,56 @@
+# Generated by Django 3.2.18 on 2023-03-21 02:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0204_auto_20230321_0204'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='organizationaddress',
+            name='address_line_3',
+        ),
+        migrations.RemoveField(
+            model_name='organizationaddress',
+            name='county',
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_address_other',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_city',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_country',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_postal_code',
+            field=models.CharField(blank=True, max_length=10, null=True),
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_province',
+            field=models.CharField(blank=True, max_length=50, null=True),
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_representativename',
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name='organizationaddress',
+            name='attorney_street_address',
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+    ]
diff --git a/backend/api/models/OrganizationAddress.py b/backend/api/models/OrganizationAddress.py
@@ -48,12 +48,6 @@ class OrganizationAddress(Auditable, EffectiveDates):
         null=True,
         db_comment="The second line of the organization's address."
     )
-    address_line_3 = models.CharField(
-        blank=True,
-        max_length=100,
-        null=True,
-        db_comment="The third line of the organization's address."
-    )
     city = models.CharField(
         blank=True,
         max_length=100,
@@ -72,24 +66,61 @@ class OrganizationAddress(Auditable, EffectiveDates):
         null=True,
         db_comment="State or Province"
     )
-    county = models.CharField(
+    country = models.CharField(
+        blank=True,
+        max_length=100,
+        null=True,
+        db_comment="Country"
+    )
+    other = models.CharField(
+        blank=True,
+        max_length=100,
+        null=True,
+        db_comment="Other Address Details"
+    )
+    attorney_city = models.CharField(
+        blank=True,
+        max_length=100,
+        null=True,
+        db_comment="City"
+    )
+    attorney_postal_code = models.CharField(
+        blank=True,
+        max_length=10,
+        null=True,
+        db_comment="Postal Code"
+    )
+    attorney_province = models.CharField(
         blank=True,
         max_length=50,
         null=True,
-        db_comment="County Name"
+        db_comment="Province"
     )
-    country = models.CharField(
+    attorney_country = models.CharField(
         blank=True,
         max_length=100,
         null=True,
-        db_comment="Country"
+        db_comment="Attorney Country"
     )
-    other = models.CharField(
+    attorney_address_other = models.CharField(
         blank=True,
         max_length=100,
         null=True,
         db_comment="Other Address Details"
     )
+    attorney_street_address = models.CharField(
+        blank=True,
+        max_length=500,
+        null=True,
+        db_comment="Street Addrees PO Box"
+    )
+    attorney_representativename = models.CharField(
+        blank=True,
+        max_length=500,
+        null=True,
+        db_comment="Representative Name"
+    )
+
 
     class Meta:
         db_table = 'organization_address'

diff --git a/backend/api/notifications/notifications.py b/backend/api/notifications/notifications.py
@@ -139,10 +139,11 @@ def send_email_for_notification(notification: NotificationMessage):
             return
 
         email_recipient = notification.user.email
+        organization_name = notification.user.organization.name
         from email.message import EmailMessage
         import smtplib
         msg = EmailMessage()
-        msg.set_content('You have received a new notification in TFRS.\nPlease sign in to view it.')
+        msg.set_content('{organization_name} has received a new notification in TFRS.\nPlease sign in to view it.'.format(organization_name=organization_name))
         bcgov_cid = make_msgid()
         msg.add_alternative("""\
         <html>
@@ -173,7 +174,7 @@ def send_email_for_notification(notification: NotificationMessage):
         line-height: 20px;
         padding-top: 20px;
         padding-bottom: 20px;">
-        <p style="margin-left: 20px;">You have received a new notification in TFRS.</p>
+        <p style="margin-left: 20px;">{organization_name} has received a new notification in TFRS.</p>
         <p style="margin-left: 20px;">Please <a href="https://lowcarbonfuels.gov.bc.ca">sign in</a>
         to view it.</p>
         </td>
@@ -182,7 +183,7 @@ def send_email_for_notification(notification: NotificationMessage):
         </table>
         </body>
         </html>
-        """.format(bcgov_cid=bcgov_cid[1:-1]), subtype='html')
+        """.format(bcgov_cid=bcgov_cid[1:-1], organization_name=organization_name), subtype='html')
 
         with open('assets/bcgov.png', 'rb') as image:
             msg.get_payload()[1].add_related(

diff --git a/backend/api/serializers/ComplianceReport.py b/backend/api/serializers/ComplianceReport.py
@@ -605,7 +605,9 @@ def get_summary(self, obj):
         lines['25'] = lines['23'] - lines['24']
         lines['27'] = lines['25'] + lines['26']
 
-        if created_on < 2023:
+        # Penalty adjustment made by business area for
+        # 2023 and above compliance periods
+        if int(obj.compliance_period.description) <= 2022:
             lines['28'] = (lines['27'] * Decimal('-200.00')).max(Decimal(0))
         else:
             lines['28'] = (lines['27'] * Decimal('-600.00')).max(Decimal(0))
@@ -1310,7 +1312,7 @@ def update(self, instance, validated_data):
 
         if summary_data and instance.supplements_id and \
                 summary_data.get('credits_offset_b', 0) and \
-                summary_data.get('credits_offset_b', 0) > max_credit_offset:
+                summary_data.get('credits_offset_b', 0) > max_credit_offset and not self.strip_summary:
             raise (serializers.ValidationError(
                 'Insufficient available credit balance. Please adjust Line 26b.'
             ))