feat: ML Features common schema and metadata API definition (#1059)

* feature model

Signed-off-by: Allison Suarez Miranda <[email protected]>

* feature model lint

Signed-off-by: Allison Suarez Miranda <[email protected]>

* lint

Signed-off-by: Allison Suarez Miranda <[email protected]>

* added query schema

Signed-off-by: Allison Suarez Miranda <[email protected]>

* reorder

Signed-off-by: Allison Suarez Miranda <[email protected]>

* lint

Signed-off-by: Allison Suarez Miranda <[email protected]>

* metadata swagger template

Signed-off-by: Allison Suarez Miranda <[email protected]>

* stubbing out all classes and methods

Signed-off-by: Allison Suarez Miranda <[email protected]>

* Created models for tabs, stubbed out all API methods

Signed-off-by: Allison Suarez Miranda <[email protected]>

* defined feature API routes

Signed-off-by: Allison Suarez Miranda <[email protected]>

* lint

Signed-off-by: Allison Suarez Miranda <[email protected]>

* bumped common version

Signed-off-by: Allison Suarez Miranda <[email protected]>

* more specific type

Signed-off-by: Allison Suarez Miranda <[email protected]>

* workflow change

Signed-off-by: Allison Suarez Miranda <[email protected]>

* lint

Signed-off-by: Allison Suarez Miranda <[email protected]>

* isort

Signed-off-by: Allison Suarez Miranda <[email protected]>

* badge import and tag import chnages

Signed-off-by: Allison Suarez Miranda <[email protected]>

* fixed imports

Signed-off-by: Allison Suarez Miranda <[email protected]>

* small fixes

Signed-off-by: Allison Suarez Miranda <[email protected]>

* added standalone query file to common

Signed-off-by: Allison Suarez Miranda <[email protected]>

* lint

Signed-off-by: Allison Suarez Miranda <[email protected]>

.github/workflows/metadata_pull_request.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        run: pip3 install -r requirements.txt && pip3 install .[all] && pip3 install codecov
+        run: pip3 install ../common && pip3 install -r requirements.txt && pip3 install .[all] && pip3 install codecov
         working-directory: ./metadata
       - name: Run python unit tests
         run: make test

common/amundsen_common/models/badge.py

@@ -0,0 +1,15 @@
+import attr
+
+from marshmallow3_annotations.ext.attrs import AttrsSchema
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class Badge:
+    badge_name: str = attr.ib()
+    category: str = attr.ib()
+
+
+class BadgeSchema(AttrsSchema):
+    class Meta:
+        target = Badge
+        register_as_scheme = True

common/amundsen_common/models/feature.py

@@ -0,0 +1,84 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import List, Optional, Dict, Any
+
+import attr
+
+from amundsen_common.models.user import User
+from amundsen_common.models.badge import Badge
+from amundsen_common.models.tag import Tag
+from amundsen_common.models.table import Column, ProgrammaticDescription, Watermark
+from marshmallow3_annotations.ext.attrs import AttrsSchema
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class ColumnItem:
+    column_name: str
+    column_type: str
+
+
+class ColumnItemSchema(AttrsSchema):
+    class Meta:
+        target = ColumnItem
+        register_as_scheme = True
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class DataSample:
+    # Modeled after preview data model in FE
+    columns: List[ColumnItem]
+    data: List[Dict[str, Any]]
+    error_text: str
+
+
+class DataSampleSchema(AttrsSchema):
+    class Meta:
+        target = DataSample
+        register_as_scheme = True
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class Feature:
+    key: Optional[str] = attr.ib(default=None)
+    name: str
+    version: str  # ex: "1.2.0"
+    status: Optional[str]
+    feature_group: str
+    entity: str
+    data_type: Optional[str]
+    availability: List[str]
+    description: Optional[str] = attr.ib(default=None)
+    owners: List[User]
+    badges: List[Badge]
+    owner_tags: Optional[List[Tag]]  # non editable
+    tags: List[Tag]  # editable
+    programmatic_descriptions: List[ProgrammaticDescription]
+    watermarks: List[Watermark]
+    last_updated_timestamp: Optional[int]
+    created_timestamp: Optional[int]
+    partition_column: Optional[Column]
+
+
+class FeatureSchema(AttrsSchema):
+    class Meta:
+        target = Feature
+        register_as_scheme = True
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class FeatureSummary:
+    key: str  # ex: test_feature_group_name/test_feature_name/1.2.0
+    name: str
+    version: str
+    availability: List[str]
+    entity: List[str]
+    description: Optional[str] = attr.ib(default=None)
+    badges: List[Badge]
+    last_updated_timestamp: Optional[int]
+
+
+class FeatureSummarySchema(AttrsSchema):
+    class Meta:
+        target = FeatureSummary
+        register_as_scheme = True

common/amundsen_common/models/lineage.py

@@ -3,7 +3,7 @@
 
 from typing import Optional, List
 
-from amundsen_common.models.table import Badge
+from amundsen_common.models.badge import Badge
 
 import attr
 from marshmallow3_annotations.ext.attrs import AttrsSchema

common/amundsen_common/models/query.py

@@ -0,0 +1,18 @@
+from typing import Optional
+
+import attr
+
+from marshmallow3_annotations.ext.attrs import AttrsSchema
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class Query:
+    name: Optional[str]
+    text: str
+    url: Optional[str]
+
+
+class QuerySchema(AttrsSchema):
+    class Meta:
+        target = Query
+        register_as_scheme = True

common/amundsen_common/models/table.py

@@ -6,6 +6,8 @@
 import attr
 
 from amundsen_common.models.user import User
+from amundsen_common.models.badge import Badge
+from amundsen_common.models.tag import Tag
 from marshmallow3_annotations.ext.attrs import AttrsSchema
 
 
@@ -21,30 +23,6 @@ class Meta:
         register_as_scheme = True
 
 
-@attr.s(auto_attribs=True, kw_only=True)
-class Tag:
-    tag_type: str
-    tag_name: str
-
-
-class TagSchema(AttrsSchema):
-    class Meta:
-        target = Tag
-        register_as_scheme = True
-
-
-@attr.s(auto_attribs=True, kw_only=True)
-class Badge:
-    badge_name: str = attr.ib()
-    category: str = attr.ib()
-
-
-class BadgeSchema(AttrsSchema):
-    class Meta:
-        target = Badge
-        register_as_scheme = True
-
-
 @attr.s(auto_attribs=True, kw_only=True)
 class Watermark:
     watermark_type: Optional[str] = None

common/amundsen_common/models/tag.py

@@ -0,0 +1,15 @@
+import attr
+
+from marshmallow3_annotations.ext.attrs import AttrsSchema
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class Tag:
+    tag_type: str
+    tag_name: str
+
+
+class TagSchema(AttrsSchema):
+    class Meta:
+        target = Tag
+        register_as_scheme = True

common/setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '0.10.1'
+__version__ = '0.11.0'
 
 setup(
     name='amundsen-common',

metadata/metadata_service/__init__.py

@@ -21,6 +21,13 @@
                                             DashboardDescriptionAPI,
                                             DashboardDetailAPI,
                                             DashboardTagAPI)
+from metadata_service.api.feature import (FeatureBadgeAPI,
+                                          FeatureDescriptionAPI,
+                                          FeatureDetailAPI,
+                                          FeatureGenerationCodeAPI,
+                                          FeatureLineageAPI, FeatureOwnerAPI,
+                                          FeatureSampleAPI, FeatureStatsAPI,
+                                          FeatureTagAPI)
 from metadata_service.api.healthcheck import healthcheck
 from metadata_service.api.popular_tables import PopularTablesAPI
 from metadata_service.api.system import Neo4jDetailAPI, StatisticsMetricsAPI
@@ -144,6 +151,23 @@ def create_app(*, config_module_class: str) -> Flask:
                      '/dashboard/<path:id>/tag/<tag>')
     api.add_resource(DashboardBadgeAPI,
                      '/dashboard/<path:id>/badge/<badge>')
+    api.add_resource(FeatureDetailAPI, '/feature/<path:feature_uri>')
+    api.add_resource(FeatureDescriptionAPI,
+                     '/feature/<path:id>/description')
+    api.add_resource(FeatureTagAPI,
+                     '/feature/<path:id>/tag/<tag>')
+    api.add_resource(FeatureBadgeAPI,
+                     '/feature/<path:id>/badge/<badge>')
+    api.add_resource(FeatureLineageAPI,
+                     '/feature/<path:id>/lineage')
+    api.add_resource(FeatureOwnerAPI,
+                     '/feature/<path:feature_uri>/owner/<owner>')
+    api.add_resource(FeatureStatsAPI,
+                     '/feature/<path:id>/stats')
+    api.add_resource(FeatureSampleAPI,
+                     '/feature/<path:id>/sample_data')
+    api.add_resource(FeatureGenerationCodeAPI,
+                     '/feature/<path:id>/generation_code')
     app.register_blueprint(api_bp)
 
     if app.config.get('SWAGGER_ENABLED'):

metadata/metadata_service/api/feature.py

@@ -0,0 +1,131 @@
+from typing import Any, Iterable, Mapping, Union
+
+from flasgger import swag_from
+from flask_restful import Resource, reqparse
+
+from metadata_service.api.badge import BadgeCommon
+from metadata_service.api.tag import TagCommon
+from metadata_service.proxy import get_proxy_client
+
+
+class FeatureDetailAPI(Resource):
+    """
+    FeatureDetail API
+    """
+
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/feature/detail_get.yml')
+    def get(self, feature_uri: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureLineageAPI(Resource):
+
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/table/lineage_get.yml')
+    def get(self, feature_uri: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureStatsAPI(Resource):
+
+    # TODO integrate stats source for FE
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/feature/detail_get.yml')
+    def get(self, feature_uri: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureGenerationCodeAPI(Resource):
+
+    # TODO use Query common model
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/feature/detail_get.yml')
+    def get(self, feature_uri: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureSampleAPI(Resource):
+
+    # TODO use DataSample common model
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/feature/detail_get.yml')
+    def get(self, feature_uri: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureOwnerAPI(Resource):
+
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/table/owner_put.yml')
+    def put(self, table_uri: str, owner: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+    @swag_from('swagger_doc/table/owner_delete.yml')
+    def delete(self, table_uri: str, owner: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureDescriptionAPI(Resource):
+
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+
+    @swag_from('swagger_doc/common/description_get.yml')
+    def get(self, id: str) -> Iterable[Any]:
+        pass
+
+    @swag_from('swagger_doc/common/description_put.yml')
+    def put(self, id: str) -> Iterable[Any]:
+        pass
+
+
+class FeatureTagAPI(Resource):
+    """
+    Only for user tags not owner tags
+    """
+
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+        self.parser = reqparse.RequestParser()
+        self.parser.add_argument('tag_type', type=str, required=False, default='default')
+
+        self._tag_common = TagCommon(client=self.client)
+
+    @swag_from('swagger_doc/tag/tag_put.yml')
+    def put(self, id: str, tag: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+    @swag_from('swagger_doc/tag/tag_delete.yml')
+    def delete(self, id: str, tag: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+
+class FeatureBadgeAPI(Resource):
+
+    def __init__(self) -> None:
+        self.client = get_proxy_client()
+        self.parser = reqparse.RequestParser()
+        self.parser.add_argument('category', type=str, required=True)
+
+        self._badge_common = BadgeCommon(client=self.client)
+
+    @swag_from('swagger_doc/badge/badge_put.yml')
+    def put(self, id: str, badge: str) -> Iterable[Union[Mapping, int, None]]:
+        pass
+
+    @swag_from('swagger_doc/badge/badge_delete.yml')
+    def delete(self, id: str, badge: str) -> Iterable[Union[Mapping, int, None]]:
+        pass

metadata/metadata_service/api/swagger_doc/feature/detail_get.yml

@@ -0,0 +1,25 @@
+Gets feature details
+---
+tags:
+  - 'feature'
+parameters:
+  - name: feature_uri
+    in: path
+    type: string
+    schema:
+      type: string
+    required: true
+    example: 'test_feature_group_name/test_feature_name/1.2.0'
+responses:
+  200:
+    description: 'Feature details'
+    content:
+      application/json:
+        schema:
+          $ref: '#/components/schemas/FeatureDetail'
+  404:
+    description: 'Feature not found'
+    content:
+      application/json:
+        schema:
+          $ref: '#/components/schemas/ErrorResponse'