DESENG-444: Engagement Metadata API

.vscode/settings.json

@@ -2,5 +2,14 @@
   "makefile.extensionOutputFolder": "./.vscode",
   "python.defaultInterpreterPath": "${workspaceFolder}/met-api/venv/bin/python",
   "python.envFile": "${workspaceFolder}/met-api/.env",
-  "python.analysis.extraPaths": ["./met-api/src", "./met-cron/src", "./analytics-api/src"]
+  "python.analysis.extraPaths": [
+    "./analytics-api/src",
+    "./met-cron/src",
+    "./met-api/src",
+  ],
+  "pylint.args": [
+    "--load-plugins",
+    "pylint_flask_sqlalchemy",
+    "pylint_flask"],
+  "python.analysis.autoImportCompletions": true
 }

CHANGELOG.MD

@@ -1,3 +1,15 @@
+## January 19, 2024
+
+- **Feature**: Add metadata management to the API [🎟️DESENG-442](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-442)
+  - Add new models, services, and endpoints for metadata and metadata taxonomy
+  - Add test coverage for new endpoints
+  - Begin documenting new endpoints using Flask-RESTX
+  - Add new endpoints to the API blueprint
+  - Enabled pylint for flask and flask_sqlalchemy in vscode
+  - Created a new "transactional" decorator to wrap model methods in a transaction that safely rolls back on error
+  - Stub out frontend components for metadata management in preparation for the next step
+- **Task**: Continue switching staff users to db-based (rather than keycloak based) authentication
+
 ## January 15, 2024
 
 - **Task** Audit for missing unit tests [🎟️DESENG-436](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-436)
@@ -30,7 +42,7 @@
 
 ## December 28, 2023
 
-> **Feature**: Added the timeline widget. [🎟️DESENG-439](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-439)
+- **Feature**: Added the timeline widget. [🎟️DESENG-439](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-439)
 
 ## December 11, 2023
 

met-api/migrations/versions/45f89f245e3d_engagement_metadata.py

@@ -22,11 +22,10 @@ def upgrade():
     sa.Column('created_date', sa.DateTime(), nullable=False),
     sa.Column('updated_date', sa.DateTime(), nullable=True),
     sa.Column('engagement_id', sa.Integer(), nullable=False),
-    # sa.Column('project_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
     sa.Column('created_by', sa.String(length=50), nullable=True),
     sa.Column('updated_by', sa.String(length=50), nullable=True),
     sa.ForeignKeyConstraint(['engagement_id'], ['engagement.id'], ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('engagement_id')
+    sa.PrimaryKeyConstraint('engagement_id', name='pk_engagement_metadata')
     )
     # ### end Alembic commands ###
 

met-api/migrations/versions/ec0128056a33_rework_engagement_metadata.py

@@ -0,0 +1,148 @@
+"""Rework engagement metadata 
+
+Revision ID: ec0128056a33
+Revises: bd0eb0d25caf
+Create Date: 2023-12-18 18:37:08.781433
+
+"""
+from enum import auto
+from alembic import op
+from regex import F
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from flask import current_app
+
+from met_api.models.tenant import Tenant as TenantModel
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec0128056a33'
+down_revision = 'bd0eb0d25caf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('engagement_metadata_taxa',
+    sa.Column('created_date', sa.DateTime(), nullable=False),
+    sa.Column('updated_date', sa.DateTime(), nullable=True),
+    sa.Column('id', sa.Integer(), nullable=False, unique=True, autoincrement=True),
+    sa.Column('tenant_id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('description', sa.String(length=256), nullable=True),
+    sa.Column('freeform', sa.Boolean(), nullable=False),
+    sa.Column('data_type', sa.String(length=64), nullable=True),
+    sa.Column('default_value', sa.Text(), nullable=True),
+    sa.Column('one_per_engagement', sa.Boolean(), nullable=True),
+    sa.Column('position', sa.Integer(), nullable=False),
+    sa.Column('created_by', sa.String(length=50), nullable=True),
+    sa.Column('updated_by', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenant.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='pk_engagement_metadata_taxa'),
+    sa.UniqueConstraint('id', name='uq_engagement_metadata_taxa_id')
+    )
+    # remove old primary key constraint from engagement_metadata.engagement_id
+    op.drop_constraint('pk_engagement_metadata', 'engagement_metadata', type_='primary')
+    op.create_index(op.f('ix_engagement_metadata_taxa_position'), 'engagement_metadata_taxa', ['position'], unique=False)
+    op.add_column('engagement_metadata', sa.Column('id', sa.Integer(), nullable=False))
+    # add new primary key constraint on engagement_metadata.id
+    op.create_primary_key('pk_engagement_metadata', 'engagement_metadata', ['id'])
+    # add autoincrement to engagement_metadata.id by creating a sequence
+    op.execute('CREATE SEQUENCE engagement_metadata_id_seq START 1')
+    op.execute('ALTER TABLE engagement_metadata ALTER COLUMN id SET DEFAULT nextval(\'engagement_metadata_id_seq\')')
+    op.execute('ALTER SEQUENCE engagement_metadata_id_seq OWNED BY engagement_metadata.id')
+    # remove not-null constraint from engagement_metadata.engagement_id
+    op.alter_column('engagement_metadata', 'engagement_id', existing_type=sa.INTEGER(), nullable=True)
+
+    op.add_column('engagement_metadata', sa.Column('taxon_id', sa.Integer(), nullable=False))
+    op.add_column('engagement_metadata', sa.Column('value', sa.Text(), nullable=False))
+    op.create_foreign_key('fk_engagement_meta_taxon', 'engagement_metadata', 'engagement_metadata_taxa', ['taxon_id'], ['id'], ondelete='CASCADE')
+    op.drop_column('engagement_metadata', 'project_tracking_id')
+    # add default taxa for default tenant
+    default_short_name = current_app.config.get('DEFAULT_TENANT_SHORT_NAME')
+    tenant_id = TenantModel.find_by_short_name(default_short_name).id
+    taxa = [
+        {
+            'name': 'keywords',
+            'description': 'Keywords for categorizing the engagement',
+            'freeform': True,
+            'one_per_engagement': False,
+            'data_type': 'text',
+        },
+        {
+            'name': 'description',
+            'description': 'Description of the engagement',
+            'freeform': True,
+            'data_type': 'long_text',
+        },
+        {
+            'name': 'jira_ticket_url',
+            'description': 'URL of the Jira ticket for this engagement',
+            'freeform': True,
+            'data_type': 'text',
+        },
+        {
+            'name': 'pmo_project_number',
+            'description': 'PMO project number',
+            'freeform': True,
+            'data_type': 'text',
+        },
+        {
+            'name': 'engagement_category',
+            'description': 'Category of the engagement',
+            'data_type': 'text',
+            'one_per_engagement': False,
+        },
+        {
+            'name': 'engagement_method',
+            'description': 'Method of engagement',
+            'data_type': 'text',
+            'default_value': "Survey",
+            'one_per_engagement': False,
+        },
+        {
+            'name': 'language',
+            'description': 'Language of the engagement',
+            'data_type': 'text',
+            'default_value': "English",
+            'one_per_engagement': False,
+        },
+        {
+            'name': 'ministry',
+            'description': 'Ministry of the engagement',
+            'data_type': 'text',
+        }
+    ]
+    for index, taxon in enumerate(taxa):
+        op.execute(
+            sa.text('INSERT INTO engagement_metadata_taxa (tenant_id, name, description, freeform, data_type, default_value, one_per_engagement, position, created_date, updated_date) '
+                    'VALUES (:tenant_id, :name, :description, :freeform, :data_type, :default_value, :one_per_engagement, :position, now(), now())')
+            .params(
+                tenant_id=tenant_id,
+                name=taxon['name'],
+                description=taxon['description'],
+                freeform=taxon.get('freeform', False),
+                data_type=taxon['data_type'],
+                default_value=taxon.get('default_value'),
+                one_per_engagement=taxon.get('one_per_engagement', True),
+                position=index + 1,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    op.add_column('engagement_metadata', sa.Column('project_tracking_id', sa.VARCHAR(length=100), autoincrement=False, nullable=True))
+    op.alter_column('engagement_metadata', 'engagement_id', existing_type=sa.INTEGER(), nullable=False)
+    op.drop_constraint('fk_engagement_meta_taxon', 'engagement_metadata', type_='foreignkey')
+    op.drop_column('engagement_metadata', 'value')
+    op.drop_column('engagement_metadata', 'taxon_id')
+    # remove primary key constraint from engagement_metadata.id
+    op.drop_constraint('pk_engagement_metadata', 'engagement_metadata', type_='primary')
+    op.drop_column('engagement_metadata', 'id')
+    op.drop_index(op.f('ix_engagement_metadata_taxa_position'), table_name='engagement_metadata_taxa')
+    # add primary key constraint to engagement_metadata.engagement_id
+    op.create_primary_key('pk_engagement_metadata', 'engagement_metadata', ['engagement_id'])
+    op.drop_table('engagement_metadata_taxa')
+    # ### end Alembic commands ###

met-api/sample.env

@@ -24,11 +24,7 @@ ENGAGEMENT_END_TIME="5 PM"
 # Default name for the tenant. Used to initially populate the database.
 DEFAULT_TENANT_SHORT_NAME="GDX"
 DEFAULT_TENANT_NAME="Government Digital Experience Division"
-DEFAULT_TENANT_DESCRIPTION="The Government Digital Experience (GDX) Division 
- is responsible for setting standards in delivering government information and
- services digitally. Their work includes creating web content guides, ensuring
- accessibility and inclusion, and overseeing forms management and visual design
- for a better digital user experience."
+DEFAULT_TENANT_DESCRIPTION="The GDX Division helps inform digital standards for web content, accessibility, forms, and design."
 
 # Keycloak configuration.
 # Populate from 'GDX Modern Engagement Tools-installation-*.json'
@@ -37,7 +33,7 @@ KEYCLOAK_BASE_URL="" # auth-server-url
 KEYCLOAK_REALMNAME="" # realm
 MET_ADMIN_CLIENT_ID="" # resource
 MET_ADMIN_CLIENT_SECRET="" # credentials.secret
-KEYCLOAK_CONNECT_TIMEOUT= 60 # seconds
+KEYCLOAK_CONNECT_TIMEOUT="60"
 
 # JWT OIDC configuration for authentication
 # Populate from 'GDX MET web (public)-installation-*.json'
@@ -111,10 +107,11 @@ CDOGS_SERVICE_CLIENT=
 CDOGS_SERVICE_CLIENT_SECRET=
 CDOGS_TOKEN_URL=
 
-JWT_OIDC_TEST_AUDIENCE=
-JWT_OIDC_TEST_CLIENT_SECRET=
-JWT_OIDC_TEST_ISSUER=
-JWT_OIDC_TEST_ALGORITHMS=
+JWT_OIDC_TEST_AUDIENCE=met-web
+JWT_OIDC_TEST_CLIENT_SECRET="1111111111"
+JWT_OIDC_TEST_ISSUER=localhost.localdomain
+JWT_OIDC_TEST_ALGORITHMS=RS256
+JWT_OIDC_TEST_CLAIMS=realm_access.roles
 
 # Test database settings
 # If unset, uses the same settings as the main database

met-api/src/met_api/__init__.py

@@ -127,15 +127,20 @@ def build_cache(app):
 def setup_jwt_manager(app_context, jwt_manager):
     """Use flask app to configure the JWTManager to work for a particular Realm."""
 
-    def get_roles(token_info):
+    def get_roles(token_info) -> list:
         """
         Consumes a token_info dictionary and returns a list of roles.
 
         Uses a configurable path to the roles in the token_info dictionary.
         """
         role_access_path = app_context.config['JWT_CONFIG']['ROLE_CLAIM']
         for key in role_access_path.split('.'):
-            token_info = token_info.get(key, {})
+            token_info = token_info.get(key, None)
+            if token_info is None:
+                app_context.logger.warning('Unable to find role in token_info. '
+                                         'Please check your JWT_ROLE_CALLBACK '
+                                         'configuration.')
+                return []
         return token_info
 
     app_context.config['JWT_ROLE_CALLBACK'] = get_roles

met-api/src/met_api/auth.py

@@ -18,19 +18,22 @@
 from flask_jwt_oidc import JwtManager
 from flask_jwt_oidc.exceptions import AuthError
 
-jwt = (
-    JwtManager()
-)  # pylint: disable=invalid-name; lower case name as used by convention in most Flask apps
+auth_methods = { # for swagger documentation
+    'apikey': {
+        'type': 'apiKey',
+        'in': 'header',
+        'name': 'Authorization'
+    }
+}
 
-
-class Auth:  # pylint: disable=too-few-public-methods
-    """Extending JwtManager to include additional functionalities."""
+class Auth(JwtManager):  # pylint: disable=too-few-public-methods
+    """Extends the JwtManager to include additional functionalities."""
 
     @classmethod
     def require(cls, f):
         """Validate the Bearer Token."""
 
-        @jwt.requires_auth
+        @auth.requires_auth
         @wraps(f)
         def decorated(*args, **kwargs):
             g.authorization_header = request.headers.get('Authorization', None)
@@ -58,7 +61,6 @@ def decorated(*args, **kwargs):
 
         return decorated
 
-
-auth = (
+jwt = auth = (
     Auth()
 )

met-api/src/met_api/config.py

@@ -349,6 +349,9 @@ def __init__(self) -> None:
         self.KC['BASE_URL'] = os.getenv('KEYCLOAK_TEST_BASE_URL', self.KC['BASE_URL'])
         self.KC['REALMNAME'] = os.getenv('KEYCLOAK_TEST_REALMNAME', self.KC['REALMNAME'])
 
+        JWT_OIDC_TEST_CLAIMS = os.getenv('JWT_OIDC_TEST_CLAIMS', 'realm_access.roles')
+        self.JWT_CONFIG['ROLE_CLAIM'] = JWT_OIDC_TEST_CLAIMS
+
     # Propagate exceptions up to the test runner
     TESTING = env_truthy('TESTING', default=True)
 

met-api/src/met_api/models/__init__.py

@@ -49,5 +49,6 @@
 from .report_setting import ReportSetting
 from .widget_video import WidgetVideo
 from .cac_form import CACForm
+from .engagement_metadata import MetadataTaxon, EngagementMetadata
 from .widget_timeline import WidgetTimeline
 from .timeline_event import TimelineEvent

met-api/src/met_api/models/db.py

@@ -1,10 +1,13 @@
 
-"""Initilizations for db, migration and marshmallow."""
+"""Initializations for db, migration and marshmallow."""
 
+import logging
 from contextlib import contextmanager
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
+from functools import wraps
+from typing import Callable, TypeVar
 # DB initialize in __init__ file
 # db variable use for create models from here
 db = SQLAlchemy()
@@ -16,16 +19,51 @@
 # Marshmallow for database model schema
 ma = Marshmallow()
 
+class AbortTransaction(Exception):
+    """
+    An exception to be raised when a transaction should be aborted. Handled
+    gracefully in the transactional decorator. Only raise this exception
+    when already inside a transactional block.
+    """
 
-@contextmanager
-def session_scope():
-    """Provide a transactional scope around a series of operations."""
-    # Using the default session for the scope
-    session = db.session
-    try:
-        yield session
-        session.commit()
-    except Exception as e:  # noqa: B901, E722
-        print(str(e))
-        session.rollback()
-        raise
+ReturnType = TypeVar('ReturnType')
+
+def transactional(db=db, autocommit=True, end_session=False
+                  ) -> Callable[[Callable[..., ReturnType]], Callable[..., ReturnType]]:
+    """
+    A decorator to quickly make an operation transactional.
+    If there is an exception during execution, the entire session will be 
+    safely rolled back to a point in time just before the decorated function 
+    was called. If not, the session will be saved, unless autocommit is set
+    to False. This helps replace most session management boilerplate.
+
+    Args:
+        db: The database instance to be used for the transaction.
+        autocommit: A boolean indicating whether to commit the session. Default is True.
+
+    Returns:
+        The result of the wrapped function `f`.
+    """
+    def decorator(f: Callable[..., ReturnType]) -> Callable[..., ReturnType]:
+        @wraps(f)
+        def decorated_function(*args, **kwargs) -> ReturnType:
+            try:
+                result = f(*args, **kwargs)
+                if autocommit:
+                    db.session.commit()
+                else:
+                    db.session.flush()
+                return result
+            except AbortTransaction:
+                logging.info("Transaction aborted.")
+                db.session.rollback() # we meant to roll back; don't raise :)
+            except Exception as e: # all other exceptions
+                logging.exception(
+                    "An error occurred during a transaction; rolling back.")
+                db.session.rollback()
+                raise e
+            finally:
+                if end_session:
+                  db.session.close()
+        return decorated_function
+    return decorator