Merge remote-tracking branch 'upstream/main'

.github/workflows/analytics-api-ci.yml

@@ -67,6 +67,8 @@ jobs:
       KEYCLOAK_TEST_BASE_URL: "http://localhost:8081"
       KEYCLOAK_TEST_REALMNAME: "demo"
       USE_TEST_KEYCLOAK_DOCKER: "YES"
+
+      SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@localhost:5432/postgres"
 
     runs-on: ubuntu-20.04
 

CHANGELOG.MD

@@ -1,35 +1,228 @@
+## April 04, 2024
+
+- **Feature** Engagement filtering - Add filtering by taxon [🎟️DESENG-445](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-445)
+
+  - Added properties to metadata taxa to allow them to be marked as filterable.
+  - Added a new file `filter_types.py` where additional filters can be added as subquery factory functions.
+  - Added a new endpoint to the API to retrieve filterable taxa.
+  - Modified the search endpoint to allow filtering by metadata.
+  - Added schemas, data validation and unit tests for the new functionality.
+  - Updated the Metadata Management UI to allow taxa to be marked as filterable.
+    - Currently, the only two filter types are `chips_any` and `chips_all`.
+    - `chips_any`: Displays as a series of toggleable buttons ("chips"), uses the `list_match_any` subquery returning engagements with any of the selected values.
+    - `chips_all`: Similar to chips_any; uses the `list_match_all` subquery to get only engagements with ALL of the selected values.
+  - If multiple filterable taxa are selected, all the taxon filters must be met for an engagement to be returned.
+  - Updated the public-facing engagement list to allow filtering by metadata taxa. This makes use of the new API endpoint to retrieve filterable taxa.
+  - Added a new filter "drawer" to the listing page to hold these and any future filter types.
+  - (Out of scope, but related to UX work for this ticket) Fixed a display issue with the public engagements page where engagements would not take up the full width of their grid cell.
+
+- **Task**: Keycloak Unit Tests for New CSS API Integration [DESENG-508](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-508)
+  - Updated Keycloak Unit Tests for New CSS API Integration.
+
+## April 02, 2024
+
+- **Task**: DEV clean up [DESENG-503](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-503)
+  - Updated deployment configurations to add new configs.
+
+## March 28, 2024
+
+- **Bug Fix**: Feedback fixes [DESENG-524](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-524)
+  - Fixed feedback path not updating issue.
+  - Fixed feedback table grid styles.
+
+## March 27, 2024
+
+- **Bug Fix**: MET - Engagement tab does not revert the filtered out data [DESENG-525](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-525)
+  - Resetting search filter values invoke the list to reload
+- **Task**: Add missing unit tests for analytics api [DESENG-482](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-482)
+  - Added unit tests for analytics API models and API end points
+
+## March 26, 2024
+
+- **Bug Fix**: Various bugs in the survey tab [DESENG-520](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-523)
+  - Update survey components and services to fix the bugs.
+    - Removed is_template and is_hidden values from autosaving to avoid the debounce function consuming it older values
+- **Task**: Add a custom dynamic page to engagement [DESENG-501](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-501)
+  - Integrated new summary and custom content tabs into the engagement form, providing users with structured sections for managing different types of content, enhancing overall organization and user experience.
+  - Implemented functionality to allow users to add custom tabs with a variety of provided icons, along with options to edit or delete tabs as needed, granting users greater flexibility and control over their engagement content layout.
+  - Streamlined data management by transitioning content and rich content from engagement tables to dedicated content tables, while also implementing logic to automatically generate default summary and custom content upon engagement creation or tab addition, ensuring consistent content structure and user experience.
+
+## March 21, 2024
+
+- **Task**: MET Web - Replace hover & background colours [DESENG-520](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-520)
+  - Update Table and Menu Dropdown hover colors
+  - Update engagement tiles hover color.
+  - Updated all white backgrounds to use css variable.
+
+## March 19, 2024
+
+- **Task**: Add poll results to results tab [DESENG-513](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-513)
+  - Added poll results to results tab.
+  - Added poll results API.
+  - Added Unit tests.
+- **Task**: Change static english text to be able to support string translations [DESENG-467](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-467)
+  - Implemented a language selector in the public header.
+  - Incorporated logic to dynamically adjust the unauthenticated route based on the selected language and load the appropriate translation file.
+  - Enhanced all public pages to fetch static text from the translation file.
+
+## March 15, 2024
+
+- **Task**: Multi-language - Create event, subcribe_item, poll, timeline widget translation tables & API routes [DESENG-515](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-515)
+  - Added Poll answer translation API.
+  - Added Timeline Event translation API.
+  - Added Subscribe Item translation API.
+  - Added Event item translation API
+  - Added Unit tests.
+- **Task** Add "Results" page to engagements [DESENG-512](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-512)
+
+## March 08, 2024
+
+- **Task**: Multi-language - Create engagement translation table & API routes [DESENG-510](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-510)
+  - Added Engagement translation model.
+  - Added Engagement translation API.
+  - Added Unit tests.
+
+## March 07, 2024
+
+- **Task**: Remove "default_values" from metadata taxa.
+  Replace with "preset values", metadata entries that are not assigned to an engagement.
+- **Task**: Update authorization documentation in the API blueprint. Update
+  metadata management to rely on normal authorization check functions.
+- **Task**: Clean up metadata management code and tests.
+- **Task**: Add endpoint for updating metadata by taxon in bulk
+- **Feature**: Add editor for metadata taxa (admin only). [🎟️DESENG-443](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-443)
+- **Feature**: Add editor for metadata entries (available to anyone who can
+  edit an engagement). [🎟️DESENG-443](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-443)
+
+## March 06, 2024
+
+- **Task**Multi-language - Create simple widget translation tables & API routes [DESENG-514](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-514)
+  - Added Widget translation model.
+  - Added Widget translation API.
+  - Added Unit tests.
+- **Task**Multi-language - Create survey translation table & API routes [DESENG-511](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-511)
+  - Added Survey Translation model.
+  - Added Survey Translation API.
+  - Added Survey Translation tests.
+
+## March 04, 2024
+
+- **Task**Engagement "save" enhancements [DESENG-507](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-507)
+  - The existing "Save" button in the floating bar has been split into two distinct actions: "Save and Continue" and "Save and Exit".
+  - Tabs are greyed out, and widgets are disabled until the engagement is successfully saved. A helpful tool-tip has been added to inform users that the engagement needs to be saved before accessing certain features.
+  - Independent save buttons previously present in tabs, such as "Additional Details", "Settings" have been removed. Now, the floating save bar is universally employed when editing an engagement.
+- **Task**Multi-language - Create language table & API [DESENG-509](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-509)
+  - Added Language model.
+  - Added Language API.
+  - Added Unit tests.
+
+## February 27, 2024
+
+- **Bug Fix**Comments cannot be approved while reviewing [DESENG-496](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-496)
+  - Fixed by adding a missing decorator for transactional methods.
+- **Task**Enhance analytics api for Improved Readability and Maintainability [DESENG-492](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-492)
+  - Refactor analytics-api config to harmonize its structure and conventions with met-api.
+  - Ensure the sample.env file maintains consistent formatting.
+  - Adjusted the component_id column size in the comment table of the met-api to resolve an error encountered during user submissions.
+
+## February 26, 2024
+
+- **Task**Models for dynamic engagement pages [DESENG-500](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-500)
+  - Implemented endpoints for dynamic engagement pages, including summary and custom sections.
+  - Default behavior ensures that each engagement has a dynamic summary page.
+  - Introduced logic to migrate existing content and rich content for engagements to the summary table.
+- **Task**Add font awesome libraries [DESENG-490](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-490)
+  - Added related libraries to the frontend package.json.
+- **Task**Adding missing unit test [DESENG-483](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-483)
+  - Added missing unit test for components.
+
+## February 20, 2024
+
+- **Task**Upgrade the version of flask [DESENG-502](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-502)
+  - Performed a Flask version upgrade to version 2.2.5.
+
+## February 16, 2024
+
+- **Task**Make a floating save/preview bar when editing engagements [DESENG-498](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-498)
+  - Implemented a floating behavior for the save/preview buttons during engagement editing. This feature persists across all tabs but exclusively saves data for the Engagement Content tab.
+
+## February 15, 2024
+
+- **Task**Restore role assignment functionality to MET with the CSS API [DESENG-473](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-473)
+  - Utilize the CSS API for efficient management of composite roles. This involves the assignment, reassignment, or removal of users from the composite roles of TEAM_MEMBER, REVIEWER, IT_ADMIN, or IT_VIEWER.
+
+## February 09, 2024
+
+- **Task**Consolidate and re-write old migration files [DESENG-452](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-452)
+  - Deleted old migration files
+  - Created [ec0128056a33_table_until_feb_09_2024.py](met-api/migrations/versions/ec0128056a33_table_until_feb_09_2024.py) with all tables until feb 09 2024
+  - Created [37176ea4708d_data_until_feb_09_2024.py](met-api/migrations/versions/37176ea4708d_data_until_feb_09_2024.py) with all initial data until feb 09 2024
+
+## February 08, 2024
+
+- **Task**Cache CORS preflight responses with the browser for a given period of time [DESENG-484](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-484)
+  - Introduces a new configuration variable to specify the maximum age for Cross-Origin Resource Sharing (CORS)
+  - Modified the CORS preflight method to utilize this newly introduced variable.
+- **Task**Consolidate and re-write old migration files [DESENG-452](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-452)
+  - Change some foreign key field to nullbale false in model files
+  - Change `rejected_reason_other` to nullable true in `submission` model
+  - Generated new migration file based on the pending model changes which confirmed to be valid
+  - Updated Unit test of email verfication to send type to the api
+
+## February 06, 2024
+
+- **Task**Convert keycloak groups to composite roles for permission levels [DESENG-447](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-447)
+  - Commented out unit test related to Keycloak groups
+  - Changed reference of Keycloak `groups` to `roles`
+  - Commented out code related to Keycloak groups
+
+## February 06, 2024
+
+- **Task** Streamline CRON jobs [DESENG-493](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-493)
+  - Aligned the CRON configuration and sample environment files with the structure used in the Met API.
+  - Eliminated the reliance on engagement metadata within CRON jobs.
+  - Implemented necessary code adjustments to seamlessly integrate with the updated CRON configuration.
+
 ## February 05, 2024
+
 - **Task** Change "Superuser" to "Administrator" [DESENG-476](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-476)
 
 ## February 02, 2024
+
 - **Task** Updated Timeline widget icons so that the circles are more consistent. [🎟️DESENG-488](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-488)
 
 ## February 01, 2024
+
 - **Task** Change name from "Engagement Core" to "Engagement Content". [🎟️DESENG-489](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-489)
 
 ## January 29, 2024
+
 - **Task** Updated Babel Traverse library. [🎟️DESENG-474](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-474)
   - Run `npm audit fix` to update the vulnerable Babel traverse library.
 
 ## January 26, 2024
+
 - **Task** Poll Widget: Front-end. [🎟️DESENG-464](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-464)
   - Created UI for Poll Widget.
   - Updated Poll widget API and unit tests.
 
 ## January 25, 2024
+
 - **Task** Resolve issue preventing met-web from deploying on the Dev OpenShift environment. [🎟️DESENG-469](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-469)
   - Remove Epic Engage-related links and update Keycloak link.
   - Remove additional authentication method.
 
 ## January 24, 2024
+
 - **Task** Update default project type to GDX for all deployments by default. [🎟️DESENG-472](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-472)
   - Set the default project type to GDX on all continuous deployment (CD) files.
   - Removed the option to deploy to EAO.
 
 ## January 22, 2024
+
 - **Task** Poll Widget: Back-end [🎟️DESENG-463](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-463)
   - Created Database models for Widget Poll, Poll Answers, Poll Response.
-  - Created API to manage  Widget Poll, Poll Answers, Poll Response.
+  - Created API to manage Widget Poll, Poll Answers, Poll Response.
   - Created Unit tests to test the code.
 - **Task** Add missing unit tests for met api [🎟️DESENG-481](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-481)
   - Added missing unit tests for met api
@@ -69,6 +262,7 @@
 ## January 9, 2024
 
 - **Task** Improvements from Epic [🎟️DESENG-468](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-468)
+
   - Improvements to Survey Result Tracking analytics
   - New Rejection Email Template for Closed Engagements
   - Export Format for Proponent updated to be in excel format
@@ -117,9 +311,10 @@
 ## November 6, 2023
 
 - **Feature**: Switch MET to use Keycloak SSO service [🎟️DESENG-408](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-408)
+
   - Switch all role-based checks on the API to use a single callback function (`current_app.config['JWT_ROLE_CALLBACK']`)
   - Added a configurable path `JWT_ROLE_CLAIM` to indicate where your SSO instance places role information in the JWT token. If your access token looks like:
-  `{ ..., "realm_access": { "roles": [ "role1", "role2"]}}` you would set `JWT_ROLE_CLAIM=realm_access.roles`
+    `{ ..., "realm_access": { "roles": [ "role1", "role2"]}}` you would set `JWT_ROLE_CLAIM=realm_access.roles`
   - Explicitly disable single tenant mode by default to ensure correct multi-tenancy behaviour
   - Remove local Keycloak instances and configuration
   - Default to the "standard" realm for Keycloak
@@ -140,7 +335,6 @@
   - Remove one old production .env file with obsolete settings
 - Changes to DEVELOPMENT.md to reflect the current state of the project
 
-
 ## v1.0.0 - 2023-10-01
 
 - App handoff from EAO to GDX

analytics-api/requirements.txt

@@ -5,13 +5,13 @@ Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
 Flask-SQLAlchemy==2.5.1
 Flask-Script==2.0.5
-Flask==2.2.3
-Jinja2==3.0.3
+Flask==2.2.5
+Jinja2==3.1.3
 Mako==1.2.4
 MarkupSafe==2.1.2
 SQLAlchemy-Utils==0.40.0
-SQLAlchemy==1.3.24
-Werkzeug==2.2.3
+SQLAlchemy==1.4.17
+Werkzeug==2.3.8
 alembic==1.10.3
 aniso8601==9.0.1
 attrs==22.2.0
@@ -23,7 +23,7 @@ charset-normalizer==3.1.0
 click==8.1.3
 ecdsa==0.18.0
 flask-jwt-oidc==0.3.0
-flask-marshmallow==0.11.0
+flask-marshmallow==0.14.0
 flask-restx==1.1.0
 gunicorn==20.1.0
 idna==3.4
@@ -51,5 +51,5 @@ threadloop==1.0.2
 thrift==0.16.0
 tornado==6.3.2
 typing_extensions==4.5.0
-urllib3==1.26.15
+urllib3==1.26.18
 zipp==3.15.0

analytics-api/sample.env

@@ -1,18 +1,46 @@
+# GDX MET Analytics API Configuration
+# For more information on these values, please see the documentation
+# or analytics-api/src/analytics-api/config.py 
+
+# Changes Flask's run mode and the set of env vars are used to configure the app. You should not need to change this here.
 FLASK_ENV=development
 
-# local db variables
-DATABASE_USERNAME=analytics
-DATABASE_PASSWORD=analytics
-DATABASE_NAME=met
-DATABASE_HOST=localhost
-DATABASE_PORT=5432
-
-JWT_OIDC_WELL_KNOWN_CONFIG=https://localhost:8080/auth/realms/met/.well-known/openid-configuration
-JWT_OIDC_AUDIENCE=account
-JWT_OIDC_ISSUER=https://localhost:8080/auth/realms/met
-JWT_OIDC_ALGORITHMS=RS256
-JWT_OIDC_JWKS_URI=https://localhost:8080/auth/realms/met/protocol/openid-connect/certs
-JWT_OIDC_CACHING_ENABLED=True
-JWT_OIDC_JWKS_CACHE_TIMEOUT=3000000
-
-CORS_ORIGIN=http://localhost:3000,http://localhost:5000
+USE_DEBUG=True # Enable a dev-friendly debug mode
+TESTING= # Handle errors normally (False) or raise exceptions (True)
+
+# CORS Settings
+CORS_ORIGINS=http://localhost:3000,http://localhost:5000
+
+# Miscellaneous Settings
+SECRET_KEY="" # For Flask sessions. If unset, this value is randomized
+
+# Database Configuration
+DATABASE_HOST="localhost"
+DATABASE_PORT="5432"
+DATABASE_USERNAME="postgres"
+DATABASE_PASSWORD="postgres"
+DATABASE_NAME="met"
+#Default: set from above settings (this overrides them)
+SQLALCHEMY_ECHO= 
+SQLALCHEMY_TRACK_MODIFICATIONS=
+
+# Keycloak configuration.
+KEYCLOAK_BASE_URL="" # auth-server-url
+KEYCLOAK_REALMNAME="" # realm
+
+# JWT OIDC configuration for authentication
+JWT_OIDC_AUDIENCE="" # resource
+JWT_OIDC_ISSUER="" # default: constructed from base url and realm name
+JWT_OIDC_WELL_KNOWN_CONFIG="" # default: constructed from issuer
+JWT_OIDC_JWKS_URI=""          # default: constructed from issuer
+JWT_OIDC_ROLE_CLAIM=client_roles # Keycloak schema
+JWT_OIDC_CACHING_ENABLED=true # Enable caching of JWKS.
+JWT_OIDC_JWKS_CACHE_TIMEOUT=300 # Timeout for JWKS cache in seconds.
+
+# Test database settings
+# If unset, uses the same settings as the main database
+DATABASE_TEST_USERNAME=
+DATABASE_TEST_PASSWORD=
+DATABASE_TEST_NAME=
+DATABASE_TEST_HOST=
+DATABASE_TEST_PORT=

analytics-api/src/analytics_api/__init__.py

@@ -37,7 +37,7 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'development')):
     # All configuration are in config file
     app.config.from_object(get_named_config(run_mode))
 
-    CORS(app, supports_credentials=True)
+    CORS(app, origins=app.config['CORS_ORIGINS'], supports_credentials=True)
 
     # Register blueprints
     app.register_blueprint(API_BLUEPRINT)
@@ -75,8 +75,21 @@ def set_secure_headers(response):
 def setup_jwt_manager(app_context, jwt_manager):
     """Use flask app to configure the JWTManager to work for a particular Realm."""
 
-    def get_roles(a_dict):
-        return a_dict['realm_access']['roles']  # pragma: no cover
+    def get_roles(token_info) -> list:
+        """
+        Consumes a token_info dictionary and returns a list of roles.
+
+        Uses a configurable path to the roles in the token_info dictionary.
+        """
+        role_access_path = app_context.config['JWT_CONFIG']['ROLE_CLAIM']
+        for key in role_access_path.split('.'):
+            token_info = token_info.get(key, None)
+            if token_info is None:
+                app_context.logger.warning('Unable to find role in token_info. '
+                                           'Please check your JWT_ROLE_CALLBACK '
+                                           'configuration.')
+                return []
+        return token_info
 
     app_context.config['JWT_ROLE_CALLBACK'] = get_roles
     jwt_manager.init_app(app_context)