Merge remote-tracking branch 'upstream/main'

.github/workflows/analytics-api-ci.yml

@@ -28,7 +28,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/met-api-ci.yml

@@ -8,7 +8,7 @@ on:
       - "met-api/**"
   push:
     branches:
-      - main    
+      - main
 
 defaults:
   run:
@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2
@@ -65,15 +65,14 @@ jobs:
       JWT_OIDC_TEST_CLIENT_SECRET: "1111111111"
       JWT_OIDC_TEST_JWKS_CACHE_TIMEOUT: "6000"
 
-
       KEYCLOAK_ADMIN_CLIENTID: "met-admin"
       KEYCLOAK_ADMIN_SECRET: "2222222222"
       KEYCLOAK_AUTH_AUDIENCE: "met-web"
       KEYCLOAK_AUTH_CLIENT_SECRET: "1111111111"
       KEYCLOAK_BASE_URL: "http://localhost:8081/auth"
       KEYCLOAK_REALMNAME: "demo"
       USE_KEYCLOAK_DOCKER: "YES"
-    
+
       KEYCLOAK_TEST_ADMIN_CLIENTID: "met-admin"
       KEYCLOAK_TEST_ADMIN_SECRET: "2222222222"
       KEYCLOAK_TEST_AUTH_AUDIENCE: "met-web"
@@ -82,7 +81,7 @@ jobs:
       KEYCLOAK_TEST_REALMNAME: "demo"
       USE_TEST_KEYCLOAK_DOCKER: "YES"
       SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@localhost:5432/postgres"
-      
+
     runs-on: ubuntu-20.04
 
     services:
@@ -97,6 +96,10 @@ jobs:
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
+    strategy:
+      matrix:
+        python-version: [3.12]
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -116,14 +119,14 @@ jobs:
         run: |
           echo "CODECOV_BRANCH=PR_${{github.head_ref}}" >> $GITHUB_ENV
         if: github.event_name == 'pull_request'
-        
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           flags: metapi
           name: codecov-met-api
-          fail_ci_if_error: true   
-          verbose: true 
+          fail_ci_if_error: true
+          verbose: true
           override_branch: ${{env.CODECOV_BRANCH}}
           token: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/met-cron-ci.yml

@@ -29,7 +29,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/met-etl-cd.yml

@@ -32,7 +32,7 @@ env:
   OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT_REPOSITORY }}
 
 jobs:
-  met-cron-cd:
+  met-etl-cd:
     runs-on: ubuntu-20.04
 
     if: github.repository == 'bcgov/met-public'

.github/workflows/notify-api-ci.yml

@@ -28,7 +28,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

CHANGELOG.MD

@@ -1,3 +1,107 @@
+## May 7, 2024
+
+- **Feature** Ensure users can belong to more than one tenant [🎟️ DESENG-478](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-478)
+  - Tested users can belong to more than one tenant.
+  - Overide the default tenant filter for staff_user table to check with user_group_membership table.
+
+## May 6, 2024
+
+- **Feature** Auto-populate question descriptions [🎟️ DESENG-596](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-596)
+  - Upgrading the version of met-formio to version 1.0.15-rc1. This version has the changes to have a default 
+  description for the form components. The changes were pushed to met-formio by the PR: https://github.com/bcgov/met-formio/pull/25.
+
+## May 2, 2024
+
+- **Feature** Added a new landing page for tenant management [🎟️ DESENG-591](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-591)
+  - The page will be updated to list and manage tenants in upcoming tickets.
+
+## April 30, 2024
+
+- **Task** Dagster ETL error [DESENG-599](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-599)
+  - Upgrading the version of python to 3.12 for dagster user code
+  - Upgrading the dependencies to support the version of python.
+  - Fixing a bug on survey etl service.
+- **Feature** Add Super Admin role [🎟️ DESENG-557](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-557)
+  - Added a new role, `SUPER_ADMIN`, to MET.
+  - The `SUPER_ADMIN` role has the highest level of access in the MET application.
+  - The `SUPER_ADMIN` role can perform all actions in MET, including actions in other tenants.
+  - The role can be only be assigned through the single sign-on (SSO) service.
+  - This overrides any other permissions set, e.g, a Viewer user who gets the super_admin role
+    will have all the permissions of a normal super admin.
+  - Change the React app's admin role to `SUPER_ADMIN` (previously `CREATE_TENANT`).
+  - Force all UI elements to be enabled for the `SUPER_ADMIN` role.
+  - Added unit tests for the new role, and updated an existing test related to cross-tenant access.
+
+## April 29, 2024
+
+- **Task** Upgrade python to 3.12 [DESENG-466](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-466)
+  - Upgrading the version of python to 3.12
+  - Upgrading the dependencies to support the version of python.
+  - Modify code as needed if the newer version does not support the former.
+
+## April 22, 2024
+
+- **Task** Openshift - Update the buildconfig for the MET-cron job runner [🎟️ DESENG-559](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-559)
+  - Versioning now uses the openshift.io namespace
+- **Task** In-app role management [DESENG-574](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-574)
+  - Introduced new database models to manage user roles and user-to-role mappings in the Met database.
+  - Revamped the authentication validation process to rely on roles retrieved directly from the database,
+    rather than relying solely on token-based roles.
+  - Eliminated redundant or unused roles from the system.
+  - Updated unit tests to to fetch roles directly from the database instead of relying on role generation from the factory and eliminated the Keycloak mock.
+  - Renamed admin and viewer role.
+  - Removed the usage of CSS API for role management.
+
+## April 11, 2024
+
+- **Task** Multi-language - Create engagement content translation tables & API routes [DESENG-544](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-544)
+  - Created a new table for engagement content translations
+  - Created API routes and services for engagement content translations
+  - Created Unit tests for engagement content translations
+
+## April 10, 2024
+
+- **Task** Remove default taxa from GDX tenant [DESENG-578](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-578)
+  - Removed the pre-populated taxon entries from the GDX tenant.
+  - With the power of the editor interface, tenants are now expected to create their own taxon entries.
+- **Task** Update contributing guide for developers [DESENG-475](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-475)
+
+## April 09, 2024
+
+- **Bugfix**: Stop Axios from trying to be a smarty pants and sending data in a format Flask doesn't understand [🎟️ DESENG-580](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-580)
+  - This was preventing the filtering feature from working
+  - Directly serialize any metadata filters before the request to avoid Axios's new nested object serialization
+  - Deserialize it again on the Flask side
+- **Task**: MET Web - Some URLs not taking users to correct locations/timing out [DESENG-542](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-542)
+  - Created authentication provider to know the logged-in state before the App component is rendered.
+  - Optimized tenant identification from the url.
+  - Optimized path segments identification.
+- **Task**: CSS Selector specificity [🎟️ DESENG-577](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-577)
+  - Replace the `!important` flag with more specific CSS selectors, specifically within dropdowns,
+    to ensure that the correct styles are applied, and that states we have not yet designed for
+    (e.g, complex focus states) are still styled appropriately and in an accessible manner.
+  - Soon, we will be doing a more in-depth revamp of dropdowns in the app to make them more
+    accessible and user-friendly. This helps work towards that goal.
+- **Bugfix**: MET Web - No save feedback unless main content tab is selected [🎟️ DESENG-541](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-541)
+  - Implemented notifications for successful saves across all engagement tabs.
+  - Resolved issue with saving engagement summary content during engagement creation.
+  - Updated tooltip on engagement content to use MetTooltip for consistency.
+
+## April 08, 2024
+
+- **Bugfix**: Submission of rejected comments [🎟️ DESENG-527](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-527)
+  - Fixed issue where resubmitted comments would not reappear in the queue for approval.
+  - Added unit tests to ensure the issue does not reoccur.
+- **Task**: MET - Clean up outstanding dependabot PRs [DESENG-535](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-535)
+  - Bump axios from 0.26.1 to 0.28.0 in met-web. Dependabot suggested this upgrade, but it required a few minor code changes.
+  - Updated the type definition of the 'error' parameter to include a generic type '{ message?: string }' to handle the AxiosError response properly.
+
+## April 05, 2024
+
+- **Task**: Allow for any gov.bc.ca email to receive emails in MET's dev & test environments [DESENG-533](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-533)
+  - Enabled dev open shift to use GC_NOTIFY api key of test which allows all email.
+  - Added new env var `SEND_EMAIL_INTERNAL_ONLY` to allow for internal emails such as govt emails only.
+
 ## April 04, 2024
 
 - **Feature** Engagement filtering - Add filtering by taxon [🎟️DESENG-445](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-445)

CONTRIBUTING.md

@@ -4,6 +4,61 @@ Government employees, public and members of the private sector are encouraged to
 
 (If you are new to GitHub, you might start with a [basic tutorial](https://help.github.com/articles/set-up-git) and check out a more detailed guide to [pull requests](https://help.github.com/articles/using-pull-requests/).)
 
-Pull requests will be evaluated by the repository guardians on a schedule and if deemed beneficial will be committed to the master.
+Pull requests will be evaluated by the repository guardians on a schedule and if deemed beneficial will be committed to the main branch.
 
 All contributors retain the original copyright to their stuff, but by contributing to this project, you grant a world-wide, royalty-free, perpetual, irrevocable, non-exclusive, transferable license to all users **under the terms of the [license](./LICENSE) under which this project is distributed**.
+
+## Changelog
+
+Developers should update the changelog with each merge into main. The changelog should reference a JIRA ticket number or, if none is associated, provide a detailed explanation of the work. The date should also be provided. No version number is necessary. For example:
+
+```
+## April 20, 2024
+- Fixed bug that didn't allow users to post [DESENG-123](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-123)
+```
+
+Take note of the previous date in the changelog when committing. If the current date is at the top of the file, add a bullet for your changes under that date header.
+
+## Pull Requests
+
+If a high volume of changes is to be submitted as a PR, we ask that the work be segmented into multiple PRs. A common threshold is that a PR should contain 20 changed files at most. Of course it also depends on the complexity of the changes but we ask that work be broken up into smaller logical chunks whenver possible. 
+
+### Examples
+
+1. Let's say you've added 5 new files and made complex changes to 10 others. You also have 5 files that have minor changes in syntax as a result of running the linter. This would be a good place to submit a PR.
+2. You have 1 new file added and you may need to make minor changes to syntax or variable names in 25-30 other files. This would be fine to submit as a single PR.
+
+Segmenting a PR is more of an art than a science and PRs typically won't be rejected if they're larger than 20 files. How you segment may be informed by how you choose to branch in git. An example segmentation workflow is provided below.
+
+### Segmentation technique
+
+One common technique used for segmenting PRs is to create a "parent" feature branch off of main. From there, you can create many "child" branches from the parent. A PR is submitted each time you want to merge a child into its parent. A child branch will represent one logical division of the work and, as mentioned above, will contain no more than 20 changed files (depending on complexity of changes).
+
+![PR Segmentation Diagram](docs/pr-segmentation.png)
+
+With this method of segmentation, you don't need to worry about committing incomplete or broken features to main. When all child branches have been completed for the feature, the parent is then merged back into main in a final PR. This final review can contain your CHANGELOG.md entry. In the title or description of your PR, please indicate that all work has already been reviewed.
+
+__Example__: You're tasked with adding a new page to the frontend which requires read access to the DB via a new API route. You create a feature branch off of main, then a child branch off of your feature branch. You then add the new API endpoint required. You submit a PR for this work by requesting to merge your child branch into your parent feature branch. Once merged, you create another child branch off your feature branch and add the new page. You submit this work as its own PR - merging into the parent feature branch again. All work is now complete so you finish things off by submitting a PR to merge your parent feature branch into the main branch. The work has already been approved in previous reviews. You can now choose to update the Changelog. If you need to add any additional, unreviewed work, you can make a note about it in the PR description.
+
+## Merge Techniques
+
+Due to how PRs are commonly segmented in this repository, and due to its active nature, rebasing is discouraged as a way to merge changes. Traditional merge commits are preferred as they preserve history and make it easier to resolve merge conflicts.
+
+## Guidelines for reviewers
+
+Conditions for whether to Approve or Request Changes for a PR can come with a lot of nuance. We try to err on the side of Approve unless there are obvious changes needed. Below is a list of reasons to "stop" a PR. Items with a star next to them are subject to interpretation by the review. Some items may prompt to discussion between the reviewer and the submitting developer so that choices can be justified.
+
+Examples of when to Request Changes
+- Logging functions are left in that aren't required or needed later (with exceptions for error logging, i.e. console.error, etc.)
+- Commented-code is left in that isn't needed later
+- Widespread incorrect syntax use (using snake case for many variables in JS when it's not required)
+- Significant amount of repeition in code that can be condensed without much issue (code isn't DRY) *
+- Unnecessary function calls, overly verbose code *
+
+## Common Components/Utilities
+
+- `@transational`: Database method decorator. If there is an exception during execution, the entire DB session will be safely rolled back to a point in time just before the decorated function was called. If not, the session will be saved, unless autocommit is set to False. This helps replace most session management boilerplate.
+- `authorization.check_auth`: Checks a user for one or more roles and for the correct engagement.
+- `schema_utils.validate`: The most commonly used method for validating request data in resources. This method takes the request JSON data and compares it to a JSON file that acts as a schema for incoming requests of that type.
+
+

analytics-api/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.5-buster
+FROM python:3.12.2-bullseye
 
 USER root
 

analytics-api/Makefile

@@ -43,15 +43,15 @@ clean-test: ## clean test files
 	rm -fr htmlcov/
 
 build-req: clean ## Upgrade requirements
-	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	test -f venv/bin/activate || python3.12 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
 	pip install --upgrade pip ;\
 	pip install -Ur requirements/prod.txt ;\
 	pip freeze | sort > requirements.txt ;\
 	cat requirements/repo-libraries.txt >> requirements.txt ;\
 	pip install -Ur requirements/repo-libraries.txt
 
-install: clean ## Install python virtrual environment
+install: clean ## Install python virtual environment
 	test -f venv/bin/activate || python3 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
 	pip install --upgrade pip ;\

analytics-api/requirements.txt

@@ -1,55 +1,55 @@
-Flask-Caching==2.0.2
+Flask-Caching==2.1.0
 Flask-Cors==3.0.10
 Flask-Mail==0.9.1
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
-Flask-SQLAlchemy==2.5.1
+Flask-SQLAlchemy==3.0.5
 Flask-Script==2.0.5
 Flask==2.2.5
 Jinja2==3.1.3
-Mako==1.2.4
+Mako==1.3.3
 MarkupSafe==2.1.2
 SQLAlchemy-Utils==0.40.0
-SQLAlchemy==1.4.17
+SQLAlchemy==1.4.52
 Werkzeug==2.3.8
-alembic==1.10.3
+alembic==1.13.1
 aniso8601==9.0.1
-attrs==22.2.0
+attrs==23.2.0
 bcrypt==4.0.1
 blinker==1.6.1
 cachelib==0.9.0
 certifi==2023.7.22
 charset-normalizer==3.1.0
-click==8.1.3
+click==8.1.7
 ecdsa==0.18.0
 flask-jwt-oidc==0.3.0
-flask-marshmallow==0.14.0
+flask-marshmallow==1.0.0
 flask-restx==1.1.0
 gunicorn==20.1.0
 idna==3.4
 importlib-metadata==6.3.0
 importlib-resources==5.12.0
 itsdangerous==2.0.1
 jaeger-client==4.8.0
-jsonschema==4.17.3
-marshmallow-sqlalchemy==0.25.0
-marshmallow==3.19.0
+jsonschema==4.21.1
+marshmallow-sqlalchemy==1.0.0
+marshmallow==3.21.1
 opentracing==2.4.0
 packaging==23.0
 pkgutil_resolve_name==1.3.10
-psycopg2-binary==2.9.6
+psycopg2-binary==2.9.9
 pyasn1==0.4.8
 pyrsistent==0.19.3
-python-dotenv==1.0.0
+python-dotenv==1.0.1
 python-jose==3.3.0
-pytz==2023.3
+pytz==2024.1
 requests==2.31.0
 rsa==4.9
 secure==0.3.0
 six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0
 tornado==6.3.2
-typing_extensions==4.5.0
+typing_extensions==4.6.0
 urllib3==1.26.18
 zipp==3.15.0