Updating migration file for engagement metadata (#2368)

* Updating migration file for engagement metadata

.github/workflows/met-api-ci.yml

@@ -78,7 +78,7 @@ jobs:
       KEYCLOAK_TEST_ADMIN_SECRET: "2222222222"
       KEYCLOAK_TEST_AUTH_AUDIENCE: "met-web"
       KEYCLOAK_TEST_AUTH_CLIENT_SECRET: "1111111111"
-      KEYCLOAK_TEST_BASE_URL: "http://localhost:8081"
+      KEYCLOAK_TEST_BASE_URL: "http://localhost:8081/auth"
       KEYCLOAK_TEST_REALMNAME: "demo"
       USE_TEST_KEYCLOAK_DOCKER: "YES"
       SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@localhost:5432/postgres"

met-api/migrations/versions/45f89f245e3d_engagement_metadata.py

@@ -22,6 +22,7 @@ def upgrade():
     sa.Column('created_date', sa.DateTime(), nullable=False),
     sa.Column('updated_date', sa.DateTime(), nullable=True),
     sa.Column('engagement_id', sa.Integer(), nullable=False),
+    sa.Column('project_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
     sa.Column('created_by', sa.String(length=50), nullable=True),
     sa.Column('updated_by', sa.String(length=50), nullable=True),
     sa.ForeignKeyConstraint(['engagement_id'], ['engagement.id'], ondelete='CASCADE'),

met-api/migrations/versions/ec0128056a33_rework_engagement_metadata.py

@@ -41,23 +41,38 @@ def upgrade():
     sa.PrimaryKeyConstraint('id', name='pk_engagement_metadata_taxa'),
     sa.UniqueConstraint('id', name='uq_engagement_metadata_taxa_id')
     )
-    # remove old primary key constraint from engagement_metadata.engagement_id
+    # remove old data from engagement_metadata table
+    op.execute('DELETE FROM engagement_metadata')
+
+    # Drop the existing primary key constraint
     op.drop_constraint('engagement_metadata_pkey', 'engagement_metadata', type_='primary')
+
+    # Create a new index on engagement_metadata_taxa
     op.create_index(op.f('ix_engagement_metadata_taxa_position'), 'engagement_metadata_taxa', ['position'], unique=False)
+
+    # Add new columns to engagement_metadata
     op.add_column('engagement_metadata', sa.Column('id', sa.Integer(), nullable=False))
-    # add new primary key constraint on engagement_metadata.id
+    op.add_column('engagement_metadata', sa.Column('taxon_id', sa.Integer(), nullable=False))
+    op.add_column('engagement_metadata', sa.Column('value', sa.Text(), nullable=False))
+
+    # Create a new primary key constraint on the 'id' column
     op.create_primary_key('pk_engagement_metadata', 'engagement_metadata', ['id'])
-    # add autoincrement to engagement_metadata.id by creating a sequence
+
+    # Create a new sequence and set it as the default for the 'id' column
     op.execute('CREATE SEQUENCE engagement_metadata_id_seq START 1')
     op.execute('ALTER TABLE engagement_metadata ALTER COLUMN id SET DEFAULT nextval(\'engagement_metadata_id_seq\')')
     op.execute('ALTER SEQUENCE engagement_metadata_id_seq OWNED BY engagement_metadata.id')
-    # remove not-null constraint from engagement_metadata.engagement_id
+
+    # Remove the not-null constraint from 'engagement_id'
     op.alter_column('engagement_metadata', 'engagement_id', existing_type=sa.INTEGER(), nullable=True)
 
-    op.add_column('engagement_metadata', sa.Column('taxon_id', sa.Integer(), nullable=False))
-    op.add_column('engagement_metadata', sa.Column('value', sa.Text(), nullable=False))
+    # Create a foreign key constraint
     op.create_foreign_key('fk_engagement_meta_taxon', 'engagement_metadata', 'engagement_metadata_taxa', ['taxon_id'], ['id'], ondelete='CASCADE')
+
+    # Remove the 'project_tracking_id' and 'project_metadata' column
     op.drop_column('engagement_metadata', 'project_tracking_id')
+    op.drop_column('engagement_metadata', 'project_metadata')
+
     # add default taxa for default tenant
     default_short_name = current_app.config.get('DEFAULT_TENANT_SHORT_NAME')
     tenant_id = TenantModel.find_by_short_name(default_short_name).id
@@ -133,16 +148,17 @@ def upgrade():
 
 
 def downgrade():
+    op.add_column('engagement_metadata', sa.Column('project_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
     op.add_column('engagement_metadata', sa.Column('project_tracking_id', sa.VARCHAR(length=100), autoincrement=False, nullable=True))
     op.alter_column('engagement_metadata', 'engagement_id', existing_type=sa.INTEGER(), nullable=False)
     op.drop_constraint('fk_engagement_meta_taxon', 'engagement_metadata', type_='foreignkey')
-    op.drop_column('engagement_metadata', 'value')
-    op.drop_column('engagement_metadata', 'taxon_id')
     # remove primary key constraint from engagement_metadata.id
     op.drop_constraint('pk_engagement_metadata', 'engagement_metadata', type_='primary')
+    op.drop_column('engagement_metadata', 'value')
+    op.drop_column('engagement_metadata', 'taxon_id')
     op.drop_column('engagement_metadata', 'id')
     op.drop_index(op.f('ix_engagement_metadata_taxa_position'), table_name='engagement_metadata_taxa')
     # add primary key constraint to engagement_metadata.engagement_id
-    op.create_primary_key('pk_engagement_metadata', 'engagement_metadata', ['engagement_id'])
+    op.create_primary_key('engagement_metadata_pkey', 'engagement_metadata', ['engagement_id'])
     op.drop_table('engagement_metadata_taxa')
     # ### end Alembic commands ###

met-api/sample.env

@@ -121,7 +121,7 @@ DATABASE_TEST_HOST=
 DATABASE_TEST_PORT=
 
 # A keycloak server is started automatically by Pytest; there is no need to start your own instance.
-KEYCLOAK_TEST_BASE_URL="http://localhost:8081"
+KEYCLOAK_TEST_BASE_URL="http://localhost:8081/auth"
 
 # Docker database settings
 # If unset, uses the same settings as the main database

met-api/src/met_api/__init__.py

@@ -138,8 +138,8 @@ def get_roles(token_info) -> list:
             token_info = token_info.get(key, None)
             if token_info is None:
                 app_context.logger.warning('Unable to find role in token_info. '
-                                         'Please check your JWT_ROLE_CALLBACK '
-                                         'configuration.')
+                                           'Please check your JWT_ROLE_CALLBACK '
+                                           'configuration.')
                 return []
         return token_info
 

met-api/src/met_api/auth.py

@@ -18,14 +18,15 @@
 from flask_jwt_oidc import JwtManager
 from flask_jwt_oidc.exceptions import AuthError
 
-auth_methods = { # for swagger documentation
+auth_methods = {  # for swagger documentation
     'apikey': {
         'type': 'apiKey',
         'in': 'header',
         'name': 'Authorization'
     }
 }
 
+
 class Auth(JwtManager):  # pylint: disable=too-few-public-methods
     """Extends the JwtManager to include additional functionalities."""
 
@@ -61,6 +62,7 @@ def decorated(*args, **kwargs):
 
         return decorated
 
+
 jwt = auth = (
     Auth()
 )

met-api/src/met_api/models/__init__.py

@@ -49,7 +49,7 @@
 from .report_setting import ReportSetting
 from .widget_video import WidgetVideo
 from .cac_form import CACForm
-from .engagement_metadata import MetadataTaxon, EngagementMetadata
+from .engagement_metadata import EngagementMetadata, MetadataTaxon
 from .widget_timeline import WidgetTimeline
 from .timeline_event import TimelineEvent
 from .widget_poll import Poll

met-api/src/met_api/models/db.py

@@ -18,19 +18,24 @@
 # Marshmallow for database model schema
 ma = Marshmallow()
 
-class AbortTransaction(Exception):
+
+class AbortTransaction(Exception):  # noqa
     """
-    An exception to be raised when a transaction should be aborted. Handled
-    gracefully in the transactional decorator. Only raise this exception
+    An exception to be raised when a transaction should be aborted.
+
+    Handled gracefully in the transactional decorator. Only raise this exception
     when already inside a transactional block.
     """
 
+
 T = TypeVar('T')
 
+
 def transactional(database=db, autocommit=True, end_session=False
                   ) -> Callable[[Callable[..., T]], Callable[..., T]]:
     """
-    A decorator to quickly make an operation transactional.
+    Decorate an operation to quickly make it transactional.
+
     If there is an exception during execution, the entire session will be
     safely rolled back to a point in time just before the decorated function
     was called. If not, the session will be saved, unless autocommit is set
@@ -45,7 +50,7 @@ def transactional(database=db, autocommit=True, end_session=False
     """
     def decorator(f: Callable[..., T]) -> Callable[..., T]:
         @wraps(f)
-        def decorated_function(*args, **kwargs)-> Optional[T]:
+        def decorated_function(*args, **kwargs) -> Optional[T]:
             try:
                 result = f(*args, **kwargs)
                 if autocommit:
@@ -54,11 +59,11 @@ def decorated_function(*args, **kwargs)-> Optional[T]:
                     database.session.flush()
                 return result
             except AbortTransaction:
-                logging.info("Transaction aborted.")
-                database.session.rollback() # we meant to roll back; don't raise :)
-            except Exception as e: # all other exceptions
+                logging.info('Transaction aborted.')
+                database.session.rollback()  # we meant to roll back; don't raise :)
+            except Exception as e:  # noqa: B902
                 logging.exception(
-                    "An error occurred during a transaction; rolling back.")
+                    'An error occurred during a transaction; rolling back.')
                 database.session.rollback()
                 raise e
             finally:

met-api/src/met_api/models/engagement_metadata.py

@@ -1,5 +1,6 @@
 """
 The EngagementMetadata model represents a unit of metadata for an Engagement.
+
 Metadata is stored as a key-value pair, where the key is a MetadataTaxon and
 the value is a string. MetadataTaxa are used to group metadata by type, and
 determine how it is displayed in the UI.
@@ -13,43 +14,46 @@
 
 
 class EngagementMetadata(BaseModel):
-    """
-    A unit of metadata for an Engagement. Can be used to store arbitrary data.
-    """
+    """A unit of metadata for an Engagement. Can be used to store arbitrary data."""
+
     __tablename__ = 'engagement_metadata'
     id = db.Column(db.Integer, primary_key=True, nullable=False, autoincrement=True)
     engagement_id = db.Column(db.Integer,
-        db.ForeignKey('engagement.id', ondelete='CASCADE'), nullable=True, index=True)
+                              db.ForeignKey('engagement.id', ondelete='CASCADE'), nullable=True, index=True)
     engagement = db.relationship('Engagement', backref='metadata')
     taxon_id = db.Column(db.Integer,
-        db.ForeignKey('engagement_metadata_taxa.id', ondelete='CASCADE'), nullable=False, index=True)
+                         db.ForeignKey('engagement_metadata_taxa.id', ondelete='CASCADE'), nullable=False, index=True)
     taxon = db.relationship('MetadataTaxon', backref='entries')
     value = db.Column(db.Text, index=True, nullable=False)
 
     @property
     def tenant(self):
+        """Extracts the tenant details for taxon."""
         return self.taxon.tenant if self.taxon else None
 
-    # Prevent primary key and foreign keys from being updated after creation
     @validates('id')
     def validate_id(self, _, new_id):
+        """Prevent primary key and foreign keys from being updated after creation."""
         if self.id and self.id != new_id:
             raise ValueError('Cannot change own ID')
         return new_id
 
     @validates('tenant_id')
     def validate_tenant_id(self, _, new_tenant_id):
+        """Prevent update of tenant id."""
         if self.tenant_id and self.tenant_id != new_tenant_id:
             raise ValueError('Cannot change tenant_id')
         return new_tenant_id
 
     @validates('engagement_id')
     def validate_engagement_id(self, _, new_engagement_id):
+        """Prevent update of engagement id."""
         if self.engagement_id and self.engagement_id != new_engagement_id:
             raise ValueError('Cannot change engagement_id')
         return new_engagement_id
 
     def __repr__(self) -> str:
+        """Return a string representation of the EngagementMetadata."""
         if not self:
             return '<EngagementMetadata: None>'
         if (not self.engagement and not self.taxon):
@@ -61,10 +65,10 @@ def __repr__(self) -> str:
         return (f'<EngagementMetadata for eng#{self.engagement_id}: '
                 f'{self.taxon.name} = {self.value}>')
 
+
 class MetadataTaxonDataType(str, enum.Enum):
-    """
-    The data types that can be stored in a metadata property.
-    """
+    """The data types that can be stored in a metadata property."""
+
     TEXT = 'string'
     LONG_TEXT = 'long-text'
     NUMBER = 'number'
@@ -88,18 +92,20 @@ def has_value(cls, value: str) -> bool:
         """Return True if the value is a valid data type."""
         return value in cls._value2member_map_
 
+
 class MetadataTaxon(BaseModel):
     """
-    A taxon to group metadata by. Taxa determine the type of data
-    that can be stored in a property, and how it is displayed.
+    A taxon to group metadata by.
+
+    Taxa determine the type of data that can be stored in a property, and how it is displayed.
     """
 
     __tablename__ = 'engagement_metadata_taxa'
 
     id = db.Column(db.Integer, primary_key=True, unique=True, autoincrement=True)
     tenant_id = db.Column(db.Integer,
-        db.ForeignKey('tenant.id', ondelete='CASCADE'),
-        nullable=False, index=True)
+                          db.ForeignKey('tenant.id', ondelete='CASCADE'),
+                          nullable=False, index=True)
     tenant = relationship('Tenant', backref='metadata_taxa')
     name = db.Column(db.String(64), nullable=True)
     description = db.Column(db.String(256), nullable=True)
@@ -110,6 +116,7 @@ class MetadataTaxon(BaseModel):
     position = db.Column(db.Integer, nullable=False, index=True)
 
     def __init__(self, **kwargs) -> None:
+        """Initialize a new instance of the MetadataTaxon class."""
         super().__init__(**kwargs)
         if not self.data_type:
             self.data_type = 'text'
@@ -118,30 +125,32 @@ def __init__(self, **kwargs) -> None:
             max_position = MetadataTaxon.query.filter_by(tenant_id=self.tenant_id).count()
             self.position = max_position + 1
 
-    # Prevent primary key and foreign keys from being updated after creation
     @validates('id')
     def validate_id(self, _, new_id):
+        """Prevent primary key and foreign keys from being updated after creation."""
         if self.id and self.id != new_id:
             raise ValueError('Cannot change own ID')
         return new_id
 
     @validates('tenant_id')
     def validate_tenant_id(self, _, new_tenant_id):
+        """Prevent update of tenant id."""
         if self.tenant_id and self.tenant_id != new_tenant_id:
             raise ValueError('Cannot change tenant_id')
         return new_tenant_id
 
     def __repr__(self) -> str:
+        """Return a string representation of the EngagementMetadata."""
         if not self:
             return '<MetadataTaxon: None>'
         return f'<MetadataTaxon #{self.id}: {self.name}>'
 
-
     @transactional()
     def move_to_position(self, new_position: int) -> None:
         """
-        Move this taxon to a specific position within the same tenant,
-        updating positions of other taxa accordingly.
+        Move this taxon to a specific position within the same tenant.
+
+        Update positions of other taxa accordingly.
         """
         tenant_id = self.tenant_id
         current_position = self.position
@@ -165,9 +174,7 @@ def move_to_position(self, new_position: int) -> None:
 
     @transactional()
     def delete(self) -> None:
-        """
-        Remove the taxon, updating the positions of subsequent taxa within the same tenant.
-        """
+        """Remove the taxon, updating the positions of subsequent taxa within the same tenant."""
         if self is not None:
             subsequent_taxa = MetadataTaxon.query.filter(
                 MetadataTaxon.tenant_id == self.tenant_id,
@@ -182,8 +189,9 @@ def delete(self) -> None:
     @transactional()
     def reorder_taxa(cls, tenant_id: int, taxon_order: list[int]) -> None:
         """
-        Reorder all taxa within a specific tenant based on a provided list
-        of taxon IDs, setting their positions accordingly.
+        Reorder all taxa within a specific tenant based on a provided list of taxon IDs.
+
+        Setting their positions accordingly.
         """
         for index, taxon_id in enumerate(taxon_order):
             taxon = cls.query.filter_by(tenant_id=tenant_id, taxon_id=taxon_id).first()

met-api/src/met_api/resources/__init__.py

@@ -81,7 +81,7 @@
 API.add_namespace(VALUE_COMPONENTS_API)
 API.add_namespace(SHAPEFILE_API)
 API.add_namespace(TENANT_API)
-API.add_namespace(METADATA_TAXON_API,path='/tenants/<string:tenant_name>/metadata')
+API.add_namespace(METADATA_TAXON_API, path='/tenants/<string:tenant_name>/metadata')
 API.add_namespace(ENGAGEMENT_METADATA_API, path='/engagements/<int:engagement_id>/metadata')
 API.add_namespace(ENGAGEMENT_MEMBERS_API, path='/engagements/<string:engagement_id>/members')
 API.add_namespace(WIDGET_DOCUMENTS_API, path='/widgets/<string:widget_id>/documents')