Feature/deseng 444: Engagement Metadata API

.vscode/settings.json

@@ -2,5 +2,14 @@
   "makefile.extensionOutputFolder": "./.vscode",
   "python.defaultInterpreterPath": "${workspaceFolder}/met-api/venv/bin/python",
   "python.envFile": "${workspaceFolder}/met-api/.env",
-  "python.analysis.extraPaths": ["./met-api/src", "./met-cron/src", "./analytics-api/src"]
+  "python.analysis.extraPaths": [
+    "./analytics-api/src",
+    "./met-cron/src",
+    "./met-api/src",
+  ],
+  "pylint.args": [
+    "--load-plugins",
+    "pylint_flask_sqlalchemy",
+    "pylint_flask"],
+  "python.analysis.autoImportCompletions": true
 }

CHANGELOG.MD

@@ -17,6 +17,18 @@
   - Added missing unit tests for met api
   - Added unit tests for error handling for met api
 
+## January 19, 2024
+
+- **Feature**: Add metadata management to the API [🎟️DESENG-442](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-442)
+  - Add new models, services, and endpoints for metadata and metadata taxonomy
+  - Add test coverage for new endpoints
+  - Begin documenting new endpoints using Flask-RESTX
+  - Add new endpoints to the API blueprint
+  - Enabled pylint for flask and flask_sqlalchemy in vscode
+  - Created a new "transactional" decorator to wrap model methods in a transaction that safely rolls back on error
+  - Stub out frontend components for metadata management in preparation for the next step
+- **Task**: Continue switching staff users to db-based (rather than keycloak based) authentication
+
 ## January 15, 2024
 
 - **Task** Audit for missing unit tests [🎟️DESENG-436](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-436)
@@ -49,7 +61,7 @@
 
 ## December 28, 2023
 
-> **Feature**: Added the timeline widget. [🎟️DESENG-439](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-439)
+- **Feature**: Added the timeline widget. [🎟️DESENG-439](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-439)
 
 ## December 11, 2023
 

met-api/migrations/versions/45f89f245e3d_engagement_metadata.py

@@ -22,7 +22,6 @@ def upgrade():
     sa.Column('created_date', sa.DateTime(), nullable=False),
     sa.Column('updated_date', sa.DateTime(), nullable=True),
     sa.Column('engagement_id', sa.Integer(), nullable=False),
-    # sa.Column('project_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
     sa.Column('created_by', sa.String(length=50), nullable=True),
     sa.Column('updated_by', sa.String(length=50), nullable=True),
     sa.ForeignKeyConstraint(['engagement_id'], ['engagement.id'], ondelete='CASCADE'),

met-api/migrations/versions/ec0128056a33_rework_engagement_metadata.py

@@ -0,0 +1,148 @@
+"""Rework engagement metadata 
+
+Revision ID: ec0128056a33
+Revises: 08f69642b7ae
+Create Date: 2023-12-18 18:37:08.781433
+
+"""
+from enum import auto
+from alembic import op
+from regex import F
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from flask import current_app
+
+from met_api.models.tenant import Tenant as TenantModel
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec0128056a33'
+down_revision = '08f69642b7ae'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('engagement_metadata_taxa',
+    sa.Column('created_date', sa.DateTime(), nullable=False),
+    sa.Column('updated_date', sa.DateTime(), nullable=True),
+    sa.Column('id', sa.Integer(), nullable=False, unique=True, autoincrement=True),
+    sa.Column('tenant_id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('description', sa.String(length=256), nullable=True),
+    sa.Column('freeform', sa.Boolean(), nullable=False),
+    sa.Column('data_type', sa.String(length=64), nullable=True),
+    sa.Column('default_value', sa.Text(), nullable=True),
+    sa.Column('one_per_engagement', sa.Boolean(), nullable=True),
+    sa.Column('position', sa.Integer(), nullable=False),
+    sa.Column('created_by', sa.String(length=50), nullable=True),
+    sa.Column('updated_by', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenant.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='pk_engagement_metadata_taxa'),
+    sa.UniqueConstraint('id', name='uq_engagement_metadata_taxa_id')
+    )
+    # remove old primary key constraint from engagement_metadata.engagement_id
+    op.drop_constraint('engagement_metadata_pkey', 'engagement_metadata', type_='primary')
+    op.create_index(op.f('ix_engagement_metadata_taxa_position'), 'engagement_metadata_taxa', ['position'], unique=False)
+    op.add_column('engagement_metadata', sa.Column('id', sa.Integer(), nullable=False))
+    # add new primary key constraint on engagement_metadata.id
+    op.create_primary_key('pk_engagement_metadata', 'engagement_metadata', ['id'])
+    # add autoincrement to engagement_metadata.id by creating a sequence
+    op.execute('CREATE SEQUENCE engagement_metadata_id_seq START 1')
+    op.execute('ALTER TABLE engagement_metadata ALTER COLUMN id SET DEFAULT nextval(\'engagement_metadata_id_seq\')')
+    op.execute('ALTER SEQUENCE engagement_metadata_id_seq OWNED BY engagement_metadata.id')
+    # remove not-null constraint from engagement_metadata.engagement_id
+    op.alter_column('engagement_metadata', 'engagement_id', existing_type=sa.INTEGER(), nullable=True)
+
+    op.add_column('engagement_metadata', sa.Column('taxon_id', sa.Integer(), nullable=False))
+    op.add_column('engagement_metadata', sa.Column('value', sa.Text(), nullable=False))
+    op.create_foreign_key('fk_engagement_meta_taxon', 'engagement_metadata', 'engagement_metadata_taxa', ['taxon_id'], ['id'], ondelete='CASCADE')
+    op.drop_column('engagement_metadata', 'project_tracking_id')
+    # add default taxa for default tenant
+    default_short_name = current_app.config.get('DEFAULT_TENANT_SHORT_NAME')
+    tenant_id = TenantModel.find_by_short_name(default_short_name).id
+    taxa = [
+        {
+            'name': 'keywords',
+            'description': 'Keywords for categorizing the engagement',
+            'freeform': True,
+            'one_per_engagement': False,
+            'data_type': 'text',
+        },
+        {
+            'name': 'description',
+            'description': 'Description of the engagement',
+            'freeform': True,
+            'data_type': 'long_text',
+        },
+        {
+            'name': 'jira_ticket_url',
+            'description': 'URL of the Jira ticket for this engagement',
+            'freeform': True,
+            'data_type': 'text',
+        },
+        {
+            'name': 'pmo_project_number',
+            'description': 'PMO project number',
+            'freeform': True,
+            'data_type': 'text',
+        },
+        {
+            'name': 'engagement_category',
+            'description': 'Category of the engagement',
+            'data_type': 'text',
+            'one_per_engagement': False,
+        },
+        {
+            'name': 'engagement_method',
+            'description': 'Method of engagement',
+            'data_type': 'text',
+            'default_value': "Survey",
+            'one_per_engagement': False,
+        },
+        {
+            'name': 'language',
+            'description': 'Language of the engagement',
+            'data_type': 'text',
+            'default_value': "English",
+            'one_per_engagement': False,
+        },
+        {
+            'name': 'ministry',
+            'description': 'Ministry of the engagement',
+            'data_type': 'text',
+        }
+    ]
+    for index, taxon in enumerate(taxa):
+        op.execute(
+            sa.text('INSERT INTO engagement_metadata_taxa (tenant_id, name, description, freeform, data_type, default_value, one_per_engagement, position, created_date, updated_date) '
+                    'VALUES (:tenant_id, :name, :description, :freeform, :data_type, :default_value, :one_per_engagement, :position, now(), now())')
+            .params(
+                tenant_id=tenant_id,
+                name=taxon['name'],
+                description=taxon['description'],
+                freeform=taxon.get('freeform', False),
+                data_type=taxon['data_type'],
+                default_value=taxon.get('default_value'),
+                one_per_engagement=taxon.get('one_per_engagement', True),
+                position=index + 1,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    op.add_column('engagement_metadata', sa.Column('project_tracking_id', sa.VARCHAR(length=100), autoincrement=False, nullable=True))
+    op.alter_column('engagement_metadata', 'engagement_id', existing_type=sa.INTEGER(), nullable=False)
+    op.drop_constraint('fk_engagement_meta_taxon', 'engagement_metadata', type_='foreignkey')
+    op.drop_column('engagement_metadata', 'value')
+    op.drop_column('engagement_metadata', 'taxon_id')
+    # remove primary key constraint from engagement_metadata.id
+    op.drop_constraint('pk_engagement_metadata', 'engagement_metadata', type_='primary')
+    op.drop_column('engagement_metadata', 'id')
+    op.drop_index(op.f('ix_engagement_metadata_taxa_position'), table_name='engagement_metadata_taxa')
+    # add primary key constraint to engagement_metadata.engagement_id
+    op.create_primary_key('pk_engagement_metadata', 'engagement_metadata', ['engagement_id'])
+    op.drop_table('engagement_metadata_taxa')
+    # ### end Alembic commands ###

met-api/sample.env

@@ -24,11 +24,7 @@ ENGAGEMENT_END_TIME="5 PM"
 # Default name for the tenant. Used to initially populate the database.
 DEFAULT_TENANT_SHORT_NAME="GDX"
 DEFAULT_TENANT_NAME="Government Digital Experience Division"
-DEFAULT_TENANT_DESCRIPTION="The Government Digital Experience (GDX) Division 
- is responsible for setting standards in delivering government information and
- services digitally. Their work includes creating web content guides, ensuring
- accessibility and inclusion, and overseeing forms management and visual design
- for a better digital user experience."
+DEFAULT_TENANT_DESCRIPTION="The GDX Division helps inform digital standards for web content, accessibility, forms, and design."
 
 # Keycloak configuration.
 # Populate from 'GDX Modern Engagement Tools-installation-*.json'
@@ -37,7 +33,7 @@ KEYCLOAK_BASE_URL="" # auth-server-url
 KEYCLOAK_REALMNAME="" # realm
 MET_ADMIN_CLIENT_ID="" # resource
 MET_ADMIN_CLIENT_SECRET="" # credentials.secret
-KEYCLOAK_CONNECT_TIMEOUT= 60 # seconds
+KEYCLOAK_CONNECT_TIMEOUT="60"
 
 # JWT OIDC configuration for authentication
 # Populate from 'GDX MET web (public)-installation-*.json'
@@ -111,10 +107,10 @@ CDOGS_SERVICE_CLIENT=
 CDOGS_SERVICE_CLIENT_SECRET=
 CDOGS_TOKEN_URL=
 
-JWT_OIDC_TEST_AUDIENCE=
-JWT_OIDC_TEST_CLIENT_SECRET=
-JWT_OIDC_TEST_ISSUER=
-JWT_OIDC_TEST_ALGORITHMS=
+JWT_OIDC_TEST_AUDIENCE=met-web
+JWT_OIDC_TEST_CLIENT_SECRET="1111111111"
+JWT_OIDC_TEST_ISSUER=localhost.localdomain
+JWT_OIDC_TEST_ALGORITHMS=RS256
 
 # Test database settings
 # If unset, uses the same settings as the main database

met-api/src/met_api/__init__.py

@@ -127,15 +127,20 @@ def build_cache(app):
 def setup_jwt_manager(app_context, jwt_manager):
     """Use flask app to configure the JWTManager to work for a particular Realm."""
 
-    def get_roles(token_info):
+    def get_roles(token_info) -> list:
         """
         Consumes a token_info dictionary and returns a list of roles.
 
         Uses a configurable path to the roles in the token_info dictionary.
         """
         role_access_path = app_context.config['JWT_CONFIG']['ROLE_CLAIM']
         for key in role_access_path.split('.'):
-            token_info = token_info.get(key, {})
+            token_info = token_info.get(key, None)
+            if token_info is None:
+                app_context.logger.warning('Unable to find role in token_info. '
+                                         'Please check your JWT_ROLE_CALLBACK '
+                                         'configuration.')
+                return []
         return token_info
 
     app_context.config['JWT_ROLE_CALLBACK'] = get_roles

met-api/src/met_api/auth.py

@@ -18,19 +18,22 @@
 from flask_jwt_oidc import JwtManager
 from flask_jwt_oidc.exceptions import AuthError
 
-jwt = (
-    JwtManager()
-)  # pylint: disable=invalid-name; lower case name as used by convention in most Flask apps
+auth_methods = { # for swagger documentation
+    'apikey': {
+        'type': 'apiKey',
+        'in': 'header',
+        'name': 'Authorization'
+    }
+}
 
-
-class Auth:  # pylint: disable=too-few-public-methods
-    """Extending JwtManager to include additional functionalities."""
+class Auth(JwtManager):  # pylint: disable=too-few-public-methods
+    """Extends the JwtManager to include additional functionalities."""
 
     @classmethod
     def require(cls, f):
         """Validate the Bearer Token."""
 
-        @jwt.requires_auth
+        @auth.requires_auth
         @wraps(f)
         def decorated(*args, **kwargs):
             g.authorization_header = request.headers.get('Authorization', None)
@@ -58,7 +61,6 @@ def decorated(*args, **kwargs):
 
         return decorated
 
-
-auth = (
+jwt = auth = (
     Auth()
 )

met-api/src/met_api/config.py

@@ -55,12 +55,12 @@ def get_named_config(environment: Union[str, None]) -> 'Config':
     }
     try:
         print(f'Loading configuration: {environment}...')
-        return config_mapping.get(environment, ProdConfig)()
+        return config_mapping.get(environment or 'production', ProdConfig)()
     except KeyError as e:
         raise KeyError(f'Configuration "{environment}" not found.') from e
 
 
-def env_truthy(env_var, default: bool = False):
+def env_truthy(env_var, default: Union[bool, str] = False):
     """
     Return True if the environment variable is set to a truthy value.
 
@@ -198,7 +198,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str:
         'JWKS_URI': os.getenv('JWT_OIDC_JWKS_URI', f'{_issuer}/protocol/openid-connect/certs'),
         'ALGORITHMS': os.getenv('JWT_OIDC_ALGORITHMS', 'RS256'),
         'AUDIENCE': os.getenv('JWT_OIDC_AUDIENCE', 'account'),
-        'CACHING_ENABLED': str(env_truthy('JWT_OIDC_CACHING_ENABLED', 'True')),
+        'CACHING_ENABLED': str(env_truthy('JWT_OIDC_CACHING_ENABLED', True)),
         'JWKS_CACHE_TIMEOUT': int(os.getenv('JWT_OIDC_JWKS_CACHE_TIMEOUT', '300')),
         'ROLE_CLAIM': os.getenv('JWT_OIDC_ROLE_CLAIM', 'client_roles'),
     }

met-api/src/met_api/models/__init__.py

@@ -49,6 +49,7 @@
 from .report_setting import ReportSetting
 from .widget_video import WidgetVideo
 from .cac_form import CACForm
+from .engagement_metadata import MetadataTaxon, EngagementMetadata
 from .widget_timeline import WidgetTimeline
 from .timeline_event import TimelineEvent
 from .widget_poll import Poll