Skip to content

Commit

Permalink
feat: Adding TypeMetadata model in amundsen-common (#1732)
Browse files Browse the repository at this point in the history 
* Adding common model for complex columns with subtypes

Signed-off-by: Kristen Armes <[email protected]>

* Removing unnecessary optional types

Signed-off-by: Kristen Armes <[email protected]>

* Remove optional from subtype children for consistency

Signed-off-by: Kristen Armes <[email protected]>

* Using type_metadata terminology instead of subtype

Signed-off-by: Kristen Armes <[email protected]>
  • Loading branch information
@kristenarmes
kristenarmes authored Feb 26, 2022
1 parent d70564a commit 6d329e9
Show file tree
Hide file tree
Showing 8 changed files with 204 additions and 171 deletions.
19 changes: 19 additions & 0 deletions common/amundsen_common/models/table.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,24 @@ class Meta:
register_as_scheme = True


@attr.s(auto_attribs=True, kw_only=True)
class TypeMetadata:
kind: str
name: str
key: str
description: Optional[str] = None
data_type: str
sort_order: int
badges: List[Badge] = []
children: List['TypeMetadata'] = []


class TypeMetadataSchema(AttrsSchema):
class Meta:
target = TypeMetadata
register_as_scheme = True


@attr.s(auto_attribs=True, kw_only=True)
class Column:
name: str
Expand All @@ -61,6 +79,7 @@ class Column:
sort_order: int
stats: List[Stat] = []
badges: Optional[List[Badge]] = []
type_metadata: Optional[TypeMetadata] = None # Used to support complex column types


class ColumnSchema(AttrsSchema):
Expand Down
2 changes: 1 addition & 1 deletion common/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from setuptools import find_packages, setup

__version__ = '0.25.0'
__version__ = '0.26.0'


requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements-dev.txt')
Expand Down
2 changes: 1 addition & 1 deletion databuilder/databuilder/models/description_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def __init__(self,
text: Optional[str],
source: str = DEFAULT_SOURCE,
description_key: Optional[str] = None,
start_label: Optional[str] = None, # Table, Column, Schema, Subtype
start_label: Optional[str] = None, # Table, Column, Schema, Type_Metadata
start_key: Optional[str] = None,
):
"""
Expand Down
6 changes: 3 additions & 3 deletions databuilder/databuilder/models/type_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@


class TypeMetadata(abc.ABC, GraphSerializable):
NODE_LABEL = 'Subtype'
RELATION_TYPE = 'SUBTYPE'
INVERSE_RELATION_TYPE = 'SUBTYPE_OF'
NODE_LABEL = 'Type_Metadata'
RELATION_TYPE = 'TYPE_METADATA'
INVERSE_RELATION_TYPE = 'TYPE_METADATA_OF'
KIND = 'kind'
NAME = 'name'
DATA_TYPE = 'data_type'
Expand Down
2 changes: 1 addition & 1 deletion databuilder/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from setuptools import find_packages, setup

__version__ = '6.7.0'
__version__ = '6.7.1'

requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
with open(requirements_path) as requirements_file:
Expand Down
117 changes: 61 additions & 56 deletions databuilder/tests/unit/models/test_fixtures/table_metadata_fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,21 +151,22 @@
'kind:String(single)': 'array',
'name:String(single)': 'has_nested_type',
'data_type:String(single)': 'array<array<array<string>>>',
NEPTUNE_HEADER_ID: 'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_HEADER_ID: 'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_HEADER_LABEL: 'Subtype',
NEPTUNE_HEADER_LABEL: 'Type_Metadata',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
},
{
'kind:String(single)': 'array',
'name:String(single)': '_inner_',
'data_type:String(single)': 'array<array<string>>',
NEPTUNE_HEADER_ID: 'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_HEADER_ID:
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_HEADER_LABEL: 'Subtype',
NEPTUNE_HEADER_LABEL: 'Type_Metadata',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
},
Expand All @@ -174,10 +175,10 @@
'name:String(single)': '_inner_',
'data_type:String(single)': 'array<string>',
NEPTUNE_HEADER_ID:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_/_inner_',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_/_inner_',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_/_inner_',
NEPTUNE_HEADER_LABEL: 'Subtype',
NEPTUNE_HEADER_LABEL: 'Type_Metadata',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
},
Expand Down Expand Up @@ -720,128 +721,132 @@
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Column:hive://gold.test_schema1/test_table1/has_nested_type',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='SUBTYPE'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='TYPE_METADATA'
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Column:hive://gold.test_schema1/test_table1/has_nested_type',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='SUBTYPE'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='TYPE_METADATA'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/has_nested_type',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_HEADER_LABEL: 'SUBTYPE',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_HEADER_LABEL: 'TYPE_METADATA',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type',
to_vertex_id='Column:hive://gold.test_schema1/test_table1/has_nested_type',
label='SUBTYPE_OF'
label='TYPE_METADATA_OF'
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type',
to_vertex_id='Column:hive://gold.test_schema1/test_table1/has_nested_type',
label='SUBTYPE_OF'
label='TYPE_METADATA_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/has_nested_type',
NEPTUNE_HEADER_LABEL: 'SUBTYPE_OF',
NEPTUNE_HEADER_LABEL: 'TYPE_METADATA_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
],
[
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type',
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
label='SUBTYPE'
label='TYPE_METADATA'
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type',
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
label='SUBTYPE'
label='TYPE_METADATA'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_HEADER_LABEL: 'SUBTYPE',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_HEADER_LABEL: 'TYPE_METADATA',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='SUBTYPE_OF'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='TYPE_METADATA_OF'
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='SUBTYPE_OF'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
label='TYPE_METADATA_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_HEADER_LABEL: 'SUBTYPE_OF',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type',
NEPTUNE_HEADER_LABEL: 'TYPE_METADATA_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
],
[
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_/_inner_',
label='SUBTYPE'
label='TYPE_METADATA'
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_/_inner_',
label='SUBTYPE'
label='TYPE_METADATA'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_/_inner_',
NEPTUNE_HEADER_LABEL: 'SUBTYPE',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_/_inner_',
NEPTUNE_HEADER_LABEL: 'TYPE_METADATA',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_/_inner_',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
label='SUBTYPE_OF'
label='TYPE_METADATA_OF'
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
from_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_/_inner_',
to_vertex_id='Subtype:hive://gold.test_schema1/test_table1/has_nested_type'
to_vertex_id='Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_',
label='SUBTYPE_OF'
label='TYPE_METADATA_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_/_inner_',
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type'
'/type/has_nested_type/_inner_/_inner_',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Subtype:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_HEADER_LABEL: 'SUBTYPE_OF',
'Type_Metadata:hive://gold.test_schema1/test_table1/has_nested_type/type/has_nested_type/_inner_',
NEPTUNE_HEADER_LABEL: 'TYPE_METADATA_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
Expand Down
Loading

0 comments on commit 6d329e9

Please sign in to comment.