From 4db1a3cdf2956d8b142306bee6da1c30d534679a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 23 Apr 2024 16:16:10 +0800 Subject: [PATCH] [#2929] improvement(pyClient): Optimize Python client code and add integration test (#3125) ### What changes were proposed in this pull request? 1. Add `_` before Python class member variable name, let it to private access level. 2. Add Metalake, Schema E2E integration test. ### Why are the changes needed? Fix: #2929 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI Passed. Co-authored-by: Xun Liu Co-authored-by: yuhui --- clients/client-python/README.md | 10 +- .../gravitino/api/catalog_change.py | 26 +-- .../client-python/gravitino/api/fileset.py | 2 +- .../gravitino/api/fileset_change.py | 76 ++++--- .../client-python/gravitino/api/metalake.py | 44 ++++ .../gravitino/api/metalake_change.py | 35 +++- clients/client-python/gravitino/api/schema.py | 4 +- .../gravitino/api/schema_change.py | 48 +++-- .../gravitino/api/supports_schemas.py | 4 +- .../gravitino/catalog/base_schema_catalog.py | 34 ++-- .../gravitino/catalog/fileset_catalog.py | 19 +- .../client/gravitino_admin_client.py | 21 +- .../gravitino/client/gravitino_client.py | 6 +- .../gravitino/client/gravitino_client_base.py | 14 +- .../gravitino/client/gravitino_metalake.py | 18 +- .../gravitino/dto/dto_converters.py | 12 +- .../gravitino/dto/fileset_dto.py | 2 +- .../gravitino/dto/metalake_dto.py | 32 ++- .../dto/requests/catalog_create_request.py | 33 +-- .../dto/requests/catalog_update_request.py | 4 +- .../dto/requests/catalog_updates_request.py | 15 +- .../dto/requests/fileset_create_request.py | 25 ++- .../dto/requests/fileset_update_request.py | 39 ++-- .../dto/requests/fileset_updates_request.py | 8 +- .../dto/requests/metalake_create_request.py | 26 ++- .../dto/requests/metalake_update_request.py | 42 ++-- .../dto/requests/metalake_updates_request.py | 13 +- .../dto/requests/schema_create_request.py | 17 +- .../dto/requests/schema_update_request.py | 27 +-- .../dto/requests/schema_updates_request.py | 16 +- .../gravitino/dto/responses/base_response.py | 11 +- .../dto/responses/catalog_list_response.py | 4 - .../dto/responses/entity_list_response.py | 7 +- .../dto/responses/fileset_response.py | 17 +- .../dto/responses/metalake_list_response.py | 17 +- .../dto/responses/metalake_response.py | 15 +- .../dto/responses/schema_response.py | 16 +- .../client-python/gravitino/dto/schema_dto.py | 35 ++-- .../gravitino/name_identifier.py | 35 ++-- clients/client-python/gravitino/namespace.py | 2 +- .../tests/integration/integration_test_env.py | 9 - .../tests/integration/test_fileset_catalog.py | 189 +++++++++--------- .../test_gravitino_admin_client.py | 116 ----------- .../tests/integration/test_metalake.py | 125 ++++++++++++ .../tests/integration/test_schema.py | 120 +++++++++++ 45 files changed, 837 insertions(+), 553 deletions(-) create mode 100644 clients/client-python/gravitino/api/metalake.py delete mode 100644 clients/client-python/tests/integration/test_gravitino_admin_client.py create mode 100644 clients/client-python/tests/integration/test_metalake.py create mode 100644 clients/client-python/tests/integration/test_schema.py diff --git a/clients/client-python/README.md b/clients/client-python/README.md index 7912065ec95..50428c7519a 100644 --- a/clients/client-python/README.md +++ b/clients/client-python/README.md @@ -16,16 +16,10 @@ ```bash pip install -e '.[dev]' ``` - -2. Run tests - ```bash - cd gravitino - ./gradlew :clients:client-python:test - ``` -3. Run integration tests +2. Run integration tests ```bash cd gravitino ./gradlew compileDistribution -x test - ./gradlew :clients:client-python:integrationTest + ./gradlew :clients:client-python:test ``` diff --git a/clients/client-python/gravitino/api/catalog_change.py b/clients/client-python/gravitino/api/catalog_change.py index f971a9b159d..71ebd04f5a6 100644 --- a/clients/client-python/gravitino/api/catalog_change.py +++ b/clients/client-python/gravitino/api/catalog_change.py @@ -66,7 +66,7 @@ class RenameCatalog: def __init__(self, new_name): self.new_name = new_name - def get_new_name(self): + def new_name(self): """Retrieves the new name set for the catalog. Returns: @@ -74,7 +74,7 @@ def get_new_name(self): """ return self.new_name - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this RenameCatalog instance with another object for equality. Two instances are considered equal if they designate the same new name for the catalog. @@ -112,7 +112,7 @@ class UpdateCatalogComment: def __init__(self, new_comment): self.new_comment = new_comment - def get_new_comment(self): + def new_comment(self): """Retrieves the new comment intended for the catalog. Returns: @@ -120,7 +120,7 @@ def get_new_comment(self): """ return self.new_comment - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this UpdateCatalogComment instance with another object for equality. Two instances are considered equal if they designate the same new comment for the catalog. @@ -159,7 +159,7 @@ def __init__(self, property, value): self.property = property self.value = value - def get_property(self): + def property(self): """Retrieves the name of the property being set in the catalog. Returns: @@ -167,7 +167,7 @@ def get_property(self): """ return self.property - def get_value(self): + def value(self): """Retrieves the value assigned to the property in the catalog. Returns: @@ -175,7 +175,7 @@ def get_value(self): """ return self.value - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this SetProperty instance with another object for equality. Two instances are considered equal if they have the same property and value for the catalog. @@ -211,7 +211,7 @@ class RemoveProperty: """A catalog change to remove a property from the catalog.""" def __init__(self, property): - self.property = property + self._property = property def get_property(self): """Retrieves the name of the property to be removed from the catalog. @@ -219,9 +219,9 @@ def get_property(self): Returns: The name of the property for removal. """ - return self.property + return self._property - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this RemoveProperty instance with another object for equality. Two instances are considered equal if they target the same property for removal from the catalog. @@ -233,7 +233,7 @@ def __eq__(self, other): """ if not isinstance(other, CatalogChange.RemoveProperty): return False - return self.property == other.property + return self._property == other._property def __hash__(self): """Generates a hash code for this RemoveProperty instance. @@ -242,7 +242,7 @@ def __hash__(self): Returns: A hash code value for this property removal operation. """ - return hash(self.property) + return hash(self._property) def __str__(self): """Provides a string representation of the RemoveProperty instance. @@ -251,4 +251,4 @@ def __str__(self): Returns: A string summary of the property removal operation. """ - return f"REMOVEPROPERTY {self.property}" + return f"REMOVEPROPERTY {self._property}" diff --git a/clients/client-python/gravitino/api/fileset.py b/clients/client-python/gravitino/api/fileset.py index 213257931af..c187d3680e0 100644 --- a/clients/client-python/gravitino/api/fileset.py +++ b/clients/client-python/gravitino/api/fileset.py @@ -10,7 +10,7 @@ class Fileset(Auditable): - """An interface representing a fileset under a schema {@link Namespace}. A fileset is a virtual + """An interface representing a fileset under a schema Namespace. A fileset is a virtual concept of the file or directory that is managed by Gravitino. Users can create a fileset object to manage the non-tabular data on the FS-like storage. The typical use case is to manage the training data for AI workloads. The major difference compare to the relational table is that the diff --git a/clients/client-python/gravitino/api/fileset_change.py b/clients/client-python/gravitino/api/fileset_change.py index 82604ed7ce7..af4c6c02201 100644 --- a/clients/client-python/gravitino/api/fileset_change.py +++ b/clients/client-python/gravitino/api/fileset_change.py @@ -3,6 +3,9 @@ This software is licensed under the Apache License version 2. """ from abc import ABC +from dataclasses import field + +from dataclasses_json import config class FilesetChange(ABC): @@ -62,16 +65,18 @@ def remove_property(property): class RenameFileset: """A fileset change to rename the fileset.""" + _new_name: str = field(metadata=config(field_name='new_name')) + def __init__(self, new_name): - self.new_name = new_name + self._new_name = new_name - def get_new_name(self): + def new_name(self): """Retrieves the new name set for the fileset. Returns: The new name of the fileset. """ - return self.new_name + return self._new_name def __eq__(self, other): """Compares this RenameFileset instance with another object for equality. @@ -85,7 +90,7 @@ def __eq__(self, other): """ if not isinstance(other, FilesetChange.RenameFileset): return False - return self.new_name == other.new_name + return self._new_name == other.new_name() def __hash__(self): """Generates a hash code for this RenameFileset instance. @@ -94,7 +99,7 @@ def __hash__(self): Returns: A hash code value for this renaming operation. """ - return hash(self.new_name) + return hash(self._new_name) def __str__(self): """Provides a string representation of the RenameFile instance. @@ -103,23 +108,25 @@ def __str__(self): Returns: A string summary of this renaming operation. """ - return f"RENAMEFILESET {self.new_name}" + return f"RENAMEFILESET {self._new_name}" class UpdateFilesetComment: """A fileset change to update the fileset comment.""" + _new_comment: str = field(metadata=config(field_name='new_comment')) + def __init__(self, new_comment): - self.new_comment = new_comment + self._new_comment = new_comment - def get_new_comment(self): + def new_comment(self): """Retrieves the new comment intended for the fileset. Returns: The new comment that has been set for the fileset. """ - return self.new_comment + return self._new_comment - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this UpdateFilesetComment instance with another object for equality. Two instances are considered equal if they designate the same new comment for the fileset. @@ -131,7 +138,7 @@ def __eq__(self, other): """ if not isinstance(other, FilesetChange.UpdateFilesetComment): return False - return self.new_comment == other.new_comment + return self._new_comment == other.new_comment() def __hash__(self): """Generates a hash code for this UpdateFileComment instance. @@ -140,7 +147,7 @@ def __hash__(self): Returns: A hash code representing this comment update operation. """ - return hash(self.new_comment) + return hash(self._new_comment) def __str__(self): """Provides a string representation of the UpdateFilesetComment instance. @@ -149,32 +156,35 @@ def __str__(self): Returns: A string summary of this comment update operation. """ - return f"UPDATEFILESETCOMMENT {self.new_comment}" + return f"UPDATEFILESETCOMMENT {self._new_comment}" class SetProperty: """A fileset change to set the property and value for the fileset.""" - def __init__(self, property, value): - self.property = property - self.value = value + _property: str = field(metadata=config(field_name='property')) + _value: str = field(metadata=config(field_name='value')) - def get_property(self): + def __init__(self, property: str, value: str): + self._property = property + self._value = value + + def property(self): """Retrieves the name of the property being set in the fileset. Returns: The name of the property. """ - return self.property + return self._property - def get_value(self): + def value(self): """Retrieves the value assigned to the property in the fileset. Returns: The value of the property. """ - return self.value + return self._value - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this SetProperty instance with another object for equality. Two instances are considered equal if they have the same property and value for the fileset. @@ -186,7 +196,7 @@ def __eq__(self, other): """ if not isinstance(other, FilesetChange.SetProperty): return False - return self.property == other.property and self.value == other.value + return self._property == other.property() and self._value == other.value() def __hash__(self): """Generates a hash code for this SetProperty instance. @@ -195,7 +205,7 @@ def __hash__(self): Returns: A hash code value for this property setting. """ - return hash((self.property, self.value)) + return hash((self._property, self._value)) def __str__(self): """Provides a string representation of the SetProperty instance. @@ -204,23 +214,25 @@ def __str__(self): Returns: A string summary of the property setting. """ - return f"SETPROPERTY {self.property} {self.value}" + return f"SETPROPERTY {self._property} {self._value}" class RemoveProperty: """A fileset change to remove a property from the fileset.""" - def __init__(self, property): - self.property = property + _property: str = field(metadata=config(field_name='property')) - def get_property(self): + def __init__(self, property: str): + self._property = property + + def property(self): """Retrieves the name of the property to be removed from the fileset. Returns: The name of the property for removal. """ - return self.property + return self._property - def __eq__(self, other): + def __eq__(self, other) -> bool: """Compares this RemoveProperty instance with another object for equality. Two instances are considered equal if they target the same property for removal from the fileset. @@ -232,7 +244,7 @@ def __eq__(self, other): """ if not isinstance(other, FilesetChange.RemoveProperty): return False - return self.property == other.property + return self._property == other.property() def __hash__(self): """Generates a hash code for this RemoveProperty instance. @@ -241,7 +253,7 @@ def __hash__(self): Returns: A hash code value for this property removal operation. """ - return hash(self.property) + return hash(self._property) def __str__(self): """Provides a string representation of the RemoveProperty instance. @@ -250,4 +262,4 @@ def __str__(self): Returns: A string summary of the property removal operation. """ - return f"REMOVEPROPERTY {self.property}" + return f"REMOVEPROPERTY {self._property}" diff --git a/clients/client-python/gravitino/api/metalake.py b/clients/client-python/gravitino/api/metalake.py new file mode 100644 index 00000000000..63ec93e136d --- /dev/null +++ b/clients/client-python/gravitino/api/metalake.py @@ -0,0 +1,44 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from abc import abstractmethod +from typing import Optional, Dict + +from gravitino.api.auditable import Auditable + + +class Metalake(Auditable): + """ + The interface of a metalake. The metalake is the top level entity in the gravitino system, + containing a set of catalogs. + """ + + @abstractmethod + def name(self) -> str: + """The name of the metalake. + + Returns: + str: The name of the metalake. + """ + pass + + @abstractmethod + def comment(self) -> Optional[str]: + """The comment of the metalake. Note. this method will return None if the comment is not set for + this metalake. + + Returns: + Optional[str]: The comment of the metalake. + """ + pass + + @abstractmethod + def properties(self) -> Optional[Dict[str, str]]: + """The properties of the metalake. Note, this method will return None if the properties are not + set. + + Returns: + Optional[Dict[str, str]]: The properties of the metalake. + """ + pass diff --git a/clients/client-python/gravitino/api/metalake_change.py b/clients/client-python/gravitino/api/metalake_change.py index db3ad87076e..19cb090a86f 100644 --- a/clients/client-python/gravitino/api/metalake_change.py +++ b/clients/client-python/gravitino/api/metalake_change.py @@ -2,7 +2,9 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field + +from dataclasses_json import config class MetalakeChange: @@ -63,32 +65,47 @@ def remove_property(property: str) -> 'RemoveProperty': @dataclass(frozen=True) class RenameMetalake: """A metalake change to rename the metalake.""" - newName: str + _new_name: str = field(metadata=config(field_name='new_name')) + + def new_name(self) -> str: + return self._new_name def __str__(self): - return f"RENAMEMETALAKE {self.newName}" + return f"RENAMEMETALAKE {self._new_name}" @dataclass(frozen=True) class UpdateMetalakeComment: """A metalake change to update the metalake comment""" - newComment: str + _new_comment: str = field(metadata=config(field_name='new_comment')) + + def new_comment(self) -> str: + return self._new_comment def __str__(self): - return f"UPDATEMETALAKECOMMENT {self.newComment}" + return f"UPDATEMETALAKECOMMENT {self._new_comment}" @dataclass(frozen=True) class SetProperty: """A metalake change to set a property and value pair for the metalake""" - property: str - value: str + _property: str = field(metadata=config(field_name='property')) + _value: str = field(metadata=config(field_name='value')) + + def property(self) -> str: + return self._property + + def value(self) -> str: + return self._value def __str__(self): - return f"SETPROPERTY {self.property} {self.value}" + return f"SETPROPERTY {self._property} {self._value}" @dataclass(frozen=True) class RemoveProperty: """A metalake change to remove a property from the metalake""" - property: str + _property: str + + def property(self) -> str: + return self._property def __str__(self): return f"REMOVEPROPERTY {self.property}" diff --git a/clients/client-python/gravitino/api/schema.py b/clients/client-python/gravitino/api/schema.py index 7773835b2c7..92cf8cc72f2 100644 --- a/clients/client-python/gravitino/api/schema.py +++ b/clients/client-python/gravitino/api/schema.py @@ -2,13 +2,13 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Optional, Dict from gravitino.api.auditable import Auditable -class Schema(Auditable, ABC): +class Schema(Auditable): """ An interface representing a schema in the Catalog. A Schema is a basic container of relational objects, like tables, views, etc. A Schema can be self-nested, diff --git a/clients/client-python/gravitino/api/schema_change.py b/clients/client-python/gravitino/api/schema_change.py index f7cb296e5ae..99138839420 100644 --- a/clients/client-python/gravitino/api/schema_change.py +++ b/clients/client-python/gravitino/api/schema_change.py @@ -3,13 +3,16 @@ This software is licensed under the Apache License version 2. """ from abc import ABC +from dataclasses import field + +from dataclasses_json import config class SchemaChange(ABC): """NamespaceChange class to set the property and value pairs for the namespace.""" @staticmethod - def set_property(property, value): + def set_property(property: str, value: str): """SchemaChange class to set the property and value pairs for the schema. Args: @@ -22,7 +25,7 @@ def set_property(property, value): return SchemaChange.SetProperty(property, value) @staticmethod - def remove_property(property): + def remove_property(property: str): """SchemaChange class to remove a property from the schema. Args: @@ -35,25 +38,29 @@ def remove_property(property): class SetProperty: """SchemaChange class to set the property and value pairs for the schema.""" - def __init__(self, property, value): - self.property = property - self.value = value - def get_property(self): + _property: str = field(metadata=config(field_name='property')) + _value: str = field(metadata=config(field_name='value')) + + def __init__(self, property: str, value: str): + self._property = property + self._value = value + + def property(self): """Retrieves the name of the property to be set. Returns: The name of the property. """ - return self.property + return self._property - def get_value(self): + def value(self): """Retrieves the value of the property to be set. Returns: The value of the property. """ - return self.value + return self._value def __eq__(self, other): """Compares this SetProperty instance with another object for equality. @@ -67,7 +74,7 @@ def __eq__(self, other): """ if not isinstance(other, SchemaChange.SetProperty): return False - return self.property == other.property and self.value == other.value + return self._property == other.property() and self._value == other.value() def __hash__(self): """Generates a hash code for this SetProperty instance. @@ -76,7 +83,7 @@ def __hash__(self): Returns: A hash code value for this property setting. """ - return hash((self.property, self.value)) + return hash((self._property, self._value)) def __str__(self): """Provides a string representation of the SetProperty instance. @@ -85,20 +92,23 @@ def __str__(self): Returns: A string summary of the property setting. """ - return f"SETPROPERTY {self.property} {self.value}" + return f"SETPROPERTY {self._property} {self._value}" class RemoveProperty: """SchemaChange class to remove a property from the schema.""" - def __init__(self, property): - self.property = property - def get_property(self): + _property: str = field(metadata=config(field_name='property')) + + def __init__(self, property: str): + self._property = property + + def property(self): """Retrieves the name of the property to be removed. Returns: The name of the property for removal. """ - return self.property + return self._property def __eq__(self, other): """Compares this RemoveProperty instance with another object for equality. @@ -112,7 +122,7 @@ def __eq__(self, other): """ if not isinstance(other, SchemaChange.RemoveProperty): return False - return self.property == other.property + return self._property == other.property() def __hash__(self): """Generates a hash code for this RemoveProperty instance. @@ -121,7 +131,7 @@ def __hash__(self): Returns: A hash code value for this property removal operation. """ - return hash(self.property) + return hash(self._property) def __str__(self): """Provides a string representation of the RemoveProperty instance. @@ -130,4 +140,4 @@ def __str__(self): Returns: A string summary of the property removal operation. """ - return f"REMOVEPROPERTY {self.property}" + return f"REMOVEPROPERTY {self._property}" diff --git a/clients/client-python/gravitino/api/supports_schemas.py b/clients/client-python/gravitino/api/supports_schemas.py index 6f8b1ecc9a1..dfd5092db76 100644 --- a/clients/client-python/gravitino/api/supports_schemas.py +++ b/clients/client-python/gravitino/api/supports_schemas.py @@ -60,7 +60,7 @@ def schema_exists(self, ident: NameIdentifier) -> bool: return False @abstractmethod - def create_schema(self, ident: NameIdentifier, comment: Optional[str], properties: Dict[str, str]) -> Schema: + def create_schema(self, ident: NameIdentifier, comment: str, properties: Dict[str, str]) -> Schema: """Create a schema in the catalog. Args: @@ -93,7 +93,7 @@ def load_schema(self, ident: NameIdentifier) -> Schema: pass @abstractmethod - def alter_schema(self, ident: NameIdentifier, changes: List[SchemaChange]) -> Schema: + def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: """Apply the metadata change to a schema in the catalog. Args: diff --git a/clients/client-python/gravitino/catalog/base_schema_catalog.py b/clients/client-python/gravitino/catalog/base_schema_catalog.py index d002c902811..cd0b5c5e0a7 100644 --- a/clients/client-python/gravitino/catalog/base_schema_catalog.py +++ b/clients/client-python/gravitino/catalog/base_schema_catalog.py @@ -6,6 +6,7 @@ from typing import Dict from gravitino.api.catalog import Catalog +from gravitino.api.schema import Schema from gravitino.api.schema_change import SchemaChange from gravitino.api.supports_schemas import SupportsSchemas from gravitino.dto.audit_dto import AuditDTO @@ -36,7 +37,8 @@ class BaseSchemaCatalog(CatalogDTO, SupportsSchemas): def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, comment: str = None, properties: Dict[str, str] = None, audit: AuditDTO = None, rest_client: HTTPClient = None): - super().__init__(_name=name, _type=type, _provider=provider, _comment=comment, _properties=properties, _audit=audit) + super().__init__(_name=name, _type=type, _provider=provider, _comment=comment, _properties=properties, + _audit=audit) self.rest_client = rest_client def as_schemas(self): @@ -56,11 +58,11 @@ def list_schemas(self, namespace: Namespace) -> [NameIdentifier]: """ Namespace.check_schema(namespace) resp = self.rest_client.get(BaseSchemaCatalog.format_schema_request_path(namespace)) - entity_list_response = EntityListResponse.from_dict(resp.json()) + entity_list_response = EntityListResponse.from_json(resp.body, infer_missing=True) entity_list_response.validate() - return entity_list_response.idents + return entity_list_response.identifiers() - def create_schema(self, ident: NameIdentifier = None, comment: str = None, properties: Dict[str, str] = None): + def create_schema(self, ident: NameIdentifier = None, comment: str = None, properties: Dict[str, str] = None) -> Schema: """Create a new schema with specified identifier, comment and metadata. Args: @@ -83,9 +85,9 @@ def create_schema(self, ident: NameIdentifier = None, comment: str = None, prope schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() - return schema_response.schema + return schema_response.schema() - def load_schema(self, ident): + def load_schema(self, ident: NameIdentifier) -> Schema: """Load the schema with specified identifier. Args: @@ -98,13 +100,14 @@ def load_schema(self, ident): The Schema with specified identifier. """ NameIdentifier.check_schema(ident) - resp = self.rest_client.get(BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name()) + resp = self.rest_client.get( + BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name()) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() - return schema_response.schema + return schema_response.schema() - def alter_schema(self, ident, *changes): + def alter_schema(self, ident: NameIdentifier, *changes: SchemaChange) -> Schema: """Alter the schema with specified identifier by applying the changes. Args: @@ -121,12 +124,13 @@ def alter_schema(self, ident, *changes): reqs = [BaseSchemaCatalog.to_schema_update_request(change) for change in changes] updatesRequest = SchemaUpdatesRequest(reqs) updatesRequest.validate() - resp = self.rest_client.put(BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name()) + resp = self.rest_client.put( + BaseSchemaCatalog.format_schema_request_path(ident.namespace()) + "/" + ident.name(), updatesRequest) schema_response = SchemaResponse.from_json(resp.body, infer_missing=True) schema_response.validate() - return schema_response.schema + return schema_response.schema() - def drop_schema(self, ident, cascade: bool): + def drop_schema(self, ident: NameIdentifier, cascade: bool) -> bool: """Drop the schema with specified identifier. Args: @@ -148,7 +152,7 @@ def drop_schema(self, ident, cascade: bool): drop_resp.validate() return drop_resp.dropped() except Exception as e: - logger.warning("Failed to drop schema {}", ident, e) + logger.warning(f"Failed to drop schema {ident}") return False @staticmethod @@ -158,8 +162,8 @@ def format_schema_request_path(ns: Namespace): @staticmethod def to_schema_update_request(change: SchemaChange): if isinstance(change, SchemaChange.SetProperty): - return SchemaUpdateRequest.SetSchemaPropertyRequest(change.property, change.value) + return SchemaUpdateRequest.SetSchemaPropertyRequest(change.property(), change.value()) elif isinstance(change, SchemaChange.RemoveProperty): - return SchemaUpdateRequest.RemoveSchemaPropertyRequest(change.property) + return SchemaUpdateRequest.RemoveSchemaPropertyRequest(change.property()) else: raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py b/clients/client-python/gravitino/catalog/fileset_catalog.py index c7f2c730e34..60f4d3e843d 100644 --- a/clients/client-python/gravitino/catalog/fileset_catalog.py +++ b/clients/client-python/gravitino/catalog/fileset_catalog.py @@ -24,7 +24,8 @@ class FilesetCatalog(BaseSchemaCatalog): - """Fileset catalog is a catalog implementation that supports fileset like metadata operations, for + """ + Fileset catalog is a catalog implementation that supports fileset like metadata operations, for example, schemas and filesets list, creation, update and deletion. A Fileset catalog is under the metalake. """ @@ -57,7 +58,7 @@ def list_filesets(self, namespace: Namespace) -> List[NameIdentifier]: entity_list_resp = EntityListResponse.from_json(resp.body, infer_missing=True) entity_list_resp.validate() - return entity_list_resp.idents + return entity_list_resp.identifiers() def load_fileset(self, ident) -> Fileset: """Load fileset metadata by {@link NameIdentifier} from the catalog. @@ -77,7 +78,7 @@ def load_fileset(self, ident) -> Fileset: fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() - return fileset_resp.fileset + return fileset_resp.fileset() def create_fileset(self, ident: NameIdentifier, comment: str, type: Catalog.Type, storage_location: str, properties: Dict[str, str]) -> Fileset: @@ -111,7 +112,7 @@ def create_fileset(self, ident: NameIdentifier, comment: str, type: Catalog.Type fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() - return fileset_resp.fileset + return fileset_resp.fileset() def alter_fileset(self, ident, *changes) -> Fileset: """Update a fileset metadata in the catalog. @@ -137,7 +138,7 @@ def alter_fileset(self, ident, *changes) -> Fileset: fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True) fileset_resp.validate() - return fileset_resp.fileset + return fileset_resp.fileset() def drop_fileset(self, ident: NameIdentifier) -> bool: """Drop a fileset from the catalog. @@ -173,12 +174,12 @@ def format_fileset_request_path(namespace: Namespace) -> str: @staticmethod def to_fileset_update_request(change: FilesetChange): if isinstance(change, FilesetChange.RenameFileset): - return FilesetUpdateRequest.RenameFilesetRequest(change.new_name) + return FilesetUpdateRequest.RenameFilesetRequest(change.new_name()) elif isinstance(change, FilesetChange.UpdateFilesetComment): - return FilesetUpdateRequest.UpdateFilesetCommentRequest(change.new_comment) + return FilesetUpdateRequest.UpdateFilesetCommentRequest(change.new_comment()) elif isinstance(change, FilesetChange.SetProperty): - return FilesetUpdateRequest.SetFilesetPropertyRequest(change.property, change.value) + return FilesetUpdateRequest.SetFilesetPropertyRequest(change.property(), change.value()) elif isinstance(change, FilesetChange.RemoveProperty): - return FilesetUpdateRequest.RemoveFilesetPropertyRequest(change.property) + return FilesetUpdateRequest.RemoveFilesetPropertyRequest(change.property()) else: raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index 4c5853209d6..844b3e12a5a 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -8,6 +8,7 @@ from gravitino.client.gravitino_client_base import GravitinoClientBase from gravitino.client.gravitino_metalake import GravitinoMetalake from gravitino.dto.dto_converters import DTOConverters +from gravitino.dto.metalake_dto import MetalakeDTO from gravitino.dto.requests.metalake_create_request import MetalakeCreateRequest from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest from gravitino.dto.responses.drop_response import DropResponse @@ -34,11 +35,11 @@ def list_metalakes(self) -> List[GravitinoMetalake]: Returns: An array of GravitinoMetalake objects representing the Metalakes. """ - resp = self.rest_client.get(self.API_METALAKES_LIST_PATH) + resp = self._rest_client.get(self.API_METALAKES_LIST_PATH) metalake_list_resp = MetalakeListResponse.from_json(resp.body, infer_missing=True) metalake_list_resp.validate() - return [GravitinoMetalake.build(o, self.rest_client) for o in metalake_list_resp.metalakes] + return [GravitinoMetalake(o, self._rest_client) for o in metalake_list_resp.metalakes()] def create_metalake(self, ident: NameIdentifier, comment: str, properties: Dict[str, str]) -> GravitinoMetalake: """Creates a new Metalake using the Gravitino API. @@ -57,11 +58,12 @@ def create_metalake(self, ident: NameIdentifier, comment: str, properties: Dict[ req = MetalakeCreateRequest(ident.name(), comment, properties) req.validate() - resp = self.rest_client.post(self.API_METALAKES_LIST_PATH, req) + resp = self._rest_client.post(self.API_METALAKES_LIST_PATH, req) metalake_response = MetalakeResponse.from_json(resp.body, infer_missing=True) metalake_response.validate() + metalake = metalake_response.metalake() - return GravitinoMetalake.build(metalake_response.metalake, self.rest_client) + return GravitinoMetalake(metalake, self._rest_client) def alter_metalake(self, ident: NameIdentifier, *changes: MetalakeChange) -> GravitinoMetalake: """Alters a specific Metalake using the Gravitino API. @@ -75,17 +77,18 @@ def alter_metalake(self, ident: NameIdentifier, *changes: MetalakeChange) -> Gra TODO: @throws NoSuchMetalakeException If the specified Metalake does not exist. TODO: @throws IllegalArgumentException If the provided changes are invalid or not applicable. """ - NameIdentifier.check_metalake(ident) + NameIdentifier.check_metalake(ident) reqs = [DTOConverters.to_metalake_update_request(change) for change in changes] updates_request = MetalakeUpdatesRequest(reqs) updates_request.validate() - resp = self.rest_client.put(self.API_METALAKES_IDENTIFIER_PATH + ident.name(), updates_request) + resp = self._rest_client.put(self.API_METALAKES_IDENTIFIER_PATH + ident.name(), updates_request) metalake_response = MetalakeResponse.from_json(resp.body, infer_missing=True) metalake_response.validate() + metalake = metalake_response.metalake() - return GravitinoMetalake.build(metalake_response.metalake, self.rest_client) + return GravitinoMetalake(metalake, self._rest_client) def drop_metalake(self, ident: NameIdentifier) -> bool: """Drops a specific Metalake using the Gravitino API. @@ -99,10 +102,10 @@ def drop_metalake(self, ident: NameIdentifier) -> bool: NameIdentifier.check_metalake(ident) try: - resp = self.rest_client.delete(self.API_METALAKES_IDENTIFIER_PATH + ident.name()) + resp = self._rest_client.delete(self.API_METALAKES_IDENTIFIER_PATH + ident.name()) dropResponse = DropResponse.from_json(resp.body, infer_missing=True) return dropResponse.dropped() except Exception as e: - logger.warning(f"Failed to drop metalake {ident.name()}", e) + logger.warning(f"Failed to drop metalake {ident}") return False diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py index 11f764dc2a7..b022a1da974 100644 --- a/clients/client-python/gravitino/client/gravitino_client.py +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -30,7 +30,7 @@ class GravitinoClient(GravitinoClientBase): It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. """ - metalake: GravitinoMetalake + _metalake: GravitinoMetalake def __init__(self, uri: str, metalake_name: str): """Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider. @@ -44,7 +44,7 @@ def __init__(self, uri: str, metalake_name: str): NoSuchMetalakeException if the metalake with specified name does not exist. """ super().__init__(uri) - self.metalake = super().load_metalake(NameIdentifier.of(metalake_name)) + self._metalake = super().load_metalake(NameIdentifier.of(metalake_name)) def get_metalake(self) -> GravitinoMetalake: """Get the current metalake object @@ -55,7 +55,7 @@ def get_metalake(self) -> GravitinoMetalake: Returns: the GravitinoMetalake object """ - return self.metalake + return self._metalake def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: return self.get_metalake().list_catalogs(namespace) diff --git a/clients/client-python/gravitino/client/gravitino_client_base.py b/clients/client-python/gravitino/client/gravitino_client_base.py index cba3ed6d643..92b0932d09f 100644 --- a/clients/client-python/gravitino/client/gravitino_client_base.py +++ b/clients/client-python/gravitino/client/gravitino_client_base.py @@ -18,7 +18,7 @@ class GravitinoClientBase: Base class for Gravitino Java client; It uses an underlying {@link RESTClient} to send HTTP requests and receive responses from the API. """ - rest_client: HTTPClient + _rest_client: HTTPClient """The REST client to communicate with the REST server""" API_METALAKES_LIST_PATH = "api/metalakes" @@ -29,7 +29,7 @@ class GravitinoClientBase: """The REST API path prefix for load a specific metalake""" def __init__(self, uri: str): - self.rest_client = HTTPClient(uri) + self._rest_client = HTTPClient(uri) def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: """Loads a specific Metalake from the Gravitino API. @@ -46,11 +46,11 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake: NameIdentifier.check_metalake(ident) - response = self.rest_client.get(GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name()) + response = self._rest_client.get(GravitinoClientBase.API_METALAKES_IDENTIFIER_PATH + ident.name()) metalake_response = MetalakeResponse.from_json(response.body, infer_missing=True) metalake_response.validate() - return GravitinoMetalake.build(metalake_response.metalake, self.rest_client) + return GravitinoMetalake(metalake_response.metalake(), self._rest_client) def get_version(self) -> GravitinoVersion: """Retrieves the version of the Gravitino API. @@ -58,15 +58,15 @@ def get_version(self) -> GravitinoVersion: Returns: A GravitinoVersion instance representing the version of the Gravitino API. """ - resp = self.rest_client.get("api/version") + resp = self._rest_client.get("api/version") resp.validate() return GravitinoVersion(resp.get_version()) def close(self): """Closes the GravitinoClient and releases any underlying resources.""" - if self.rest_client is not None: + if self._rest_client is not None: try: - self.rest_client.close() + self._rest_client.close() except Exception as e: logger.warning("Failed to close the HTTP REST client", e) diff --git a/clients/client-python/gravitino/client/gravitino_metalake.py b/clients/client-python/gravitino/client/gravitino_metalake.py index 4d043863e7a..95b3742af89 100644 --- a/clients/client-python/gravitino/client/gravitino_metalake.py +++ b/clients/client-python/gravitino/client/gravitino_metalake.py @@ -19,7 +19,6 @@ from gravitino.namespace import Namespace from gravitino.utils import HTTPClient - from typing import List, Dict logger = logging.getLogger(__name__) @@ -48,15 +47,10 @@ class GravitinoMetalake(MetalakeDTO): API_METALAKES_CATALOGS_PATH = "api/metalakes/{}/catalogs/{}" - def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None, audit: AuditDTO = None, - rest_client: HTTPClient = None): - super().__init__(name=name, comment=comment, properties=properties, audit=audit) - self.rest_client = rest_client - - @classmethod - def build(cls, metalake: MetalakeDTO = None, client: HTTPClient = None): - return cls(name=metalake.name, comment=metalake.comment, properties=metalake.properties, - audit=metalake.audit, rest_client=client) + def __init__(self, metalake: MetalakeDTO = None, client: HTTPClient = None): + super().__init__(_name=metalake.name(), _comment=metalake.comment(), _properties=metalake.properties(), + _audit=metalake.audit_info()) + self.rest_client = client def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: """List all the catalogs under this metalake with specified namespace. @@ -75,7 +69,7 @@ def list_catalogs(self, namespace: Namespace) -> List[NameIdentifier]: response = self.rest_client.get(url) entityList = EntityListResponse.from_json(response.body, infer_missing=True) entityList.validate() - return entityList.idents + return entityList.identifiers() def list_catalogs_info(self, namespace: Namespace) -> List[Catalog]: """List all the catalogs with their information under this metalake with specified namespace. @@ -197,5 +191,5 @@ def drop_catalog(self, ident: NameIdentifier) -> bool: return drop_response.dropped() except Exception as e: - logger.warning(f"Failed to drop catalog {ident}: {e}") + logger.warning(f"Failed to drop catalog {ident}") return False diff --git a/clients/client-python/gravitino/dto/dto_converters.py b/clients/client-python/gravitino/dto/dto_converters.py index c0e66165b2b..515c0c17652 100644 --- a/clients/client-python/gravitino/dto/dto_converters.py +++ b/clients/client-python/gravitino/dto/dto_converters.py @@ -19,13 +19,13 @@ class DTOConverters: def to_metalake_update_request(change: MetalakeChange) -> object: # Assuming MetalakeUpdateRequest has similar nested class structure for requests if isinstance(change, MetalakeChange.RenameMetalake): - return MetalakeUpdateRequest.RenameMetalakeRequest(change.newName) + return MetalakeUpdateRequest.RenameMetalakeRequest(change.new_name()) elif isinstance(change, MetalakeChange.UpdateMetalakeComment): - return MetalakeUpdateRequest.UpdateMetalakeCommentRequest(change.newComment) + return MetalakeUpdateRequest.UpdateMetalakeCommentRequest(change.new_comment()) elif isinstance(change, MetalakeChange.SetProperty): - return MetalakeUpdateRequest.SetMetalakePropertyRequest(change.property, change.value) + return MetalakeUpdateRequest.SetMetalakePropertyRequest(change.property(), change.value()) elif isinstance(change, MetalakeChange.RemoveProperty): - return MetalakeUpdateRequest.RemoveMetalakePropertyRequest(change.property) + return MetalakeUpdateRequest.RemoveMetalakePropertyRequest(change.property()) else: raise ValueError(f"Unknown change type: {type(change).__name__}") @@ -49,8 +49,8 @@ def to_catalog_update_request(change: CatalogChange): elif isinstance(change, CatalogChange.UpdateCatalogComment): return CatalogUpdateRequest.UpdateCatalogCommentRequest(change.new_comment) elif isinstance(change, CatalogChange.SetProperty): - return CatalogUpdateRequest.SetCatalogPropertyRequest(change.property, change.value) + return CatalogUpdateRequest.SetCatalogPropertyRequest(change.property(), change.value()) elif isinstance(change, CatalogChange.RemoveProperty): - return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change.property) + return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change._property) else: raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/dto/fileset_dto.py b/clients/client-python/gravitino/dto/fileset_dto.py index f5540f4efe1..109443e63bb 100644 --- a/clients/client-python/gravitino/dto/fileset_dto.py +++ b/clients/client-python/gravitino/dto/fileset_dto.py @@ -20,7 +20,7 @@ class FilesetDTO(Fileset, DataClassJsonMixin): _type: Fileset.Type = field(metadata=config(field_name='type')) _properties: Dict[str, str] = field(metadata=config(field_name='properties')) _storage_location: str = field(default=None, metadata=config(field_name='storageLocation')) - _audit: AuditDTO = field(default=None) + _audit: AuditDTO = field(default=None, metadata=config(field_name='audit')) def name(self) -> str: return self._name diff --git a/clients/client-python/gravitino/dto/metalake_dto.py b/clients/client-python/gravitino/dto/metalake_dto.py index 627d772ca68..8fc08116d20 100644 --- a/clients/client-python/gravitino/dto/metalake_dto.py +++ b/clients/client-python/gravitino/dto/metalake_dto.py @@ -2,37 +2,51 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Dict -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import DataClassJsonMixin, config +from gravitino.api.audit import Audit from gravitino.dto.audit_dto import AuditDTO +from gravitino.api.metalake import Metalake @dataclass -class MetalakeDTO(DataClassJsonMixin): +class MetalakeDTO(Metalake, DataClassJsonMixin): """Represents a Metalake Data Transfer Object (DTO) that implements the Metalake interface.""" - name: str + _name: str = field(metadata=config(field_name='name')) """The name of the Metalake DTO.""" - comment: Optional[str] + _comment: Optional[str] = field(metadata=config(field_name='comment')) """The comment of the Metalake DTO.""" - properties: Optional[Dict[str, str]] = None + _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) """The properties of the Metalake DTO.""" - audit: AuditDTO = None + _audit: Optional[AuditDTO] = field(metadata=config(field_name='audit')) """The audit information of the Metalake DTO.""" + def name(self) -> str: + return self._name + + def comment(self) -> str: + return self._comment + + def properties(self) -> Dict[str, str]: + return self._properties + + def audit_info(self) -> Audit: + return self._audit + def equals(self, other): if self == other: return True if not isinstance(other, MetalakeDTO): return False - return self.name == other.name and self.comment == other.comment and \ - self.property_equal(self.properties, other.properties) and self.audit == other.audit + return self._name == other._name and self._comment == other._comment and \ + self.property_equal(self._properties, other._properties) and self._audit == other._audit def property_equal(self, p1, p2): if p1 is None and p2 is None: diff --git a/clients/client-python/gravitino/dto/requests/catalog_create_request.py b/clients/client-python/gravitino/dto/requests/catalog_create_request.py index 78b00ef0f43..ef9a35fd4b5 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_create_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_create_request.py @@ -2,30 +2,31 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Dict -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config from gravitino.api.catalog import Catalog +from gravitino.rest.rest_message import RESTRequest @dataclass -class CatalogCreateRequest(DataClassJsonMixin): +class CatalogCreateRequest(RESTRequest): """Represents a request to create a catalog.""" - name: str - type: Catalog.Type - provider: str - comment: Optional[str] - properties: Optional[Dict[str, str]] + _name: str = field(metadata=config(field_name='name')) + _type: Catalog.Type = field(metadata=config(field_name='type')) + _provider: str = field(metadata=config(field_name='provider')) + _comment: Optional[str] = field(metadata=config(field_name='comment')) + _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) def __init__(self, name: str = None, type: Catalog.Type = Catalog.Type.UNSUPPORTED, provider: str = None, comment: str = None, properties: Dict[str, str] = None): - self.name = name - self.type = type - self.provider = provider - self.comment = comment - self.properties = properties + self._name = name + self._type = type + self._provider = provider + self._comment = comment + self._properties = properties def validate(self): """Validates the fields of the request. @@ -33,6 +34,6 @@ def validate(self): Raises: IllegalArgumentException if name or type are not set. """ - assert self.name is not None, "\"name\" field is required and cannot be empty" - assert self.type is not None, "\"type\" field is required and cannot be empty" - assert self.provider is not None, "\"provider\" field is required and cannot be empty" + assert self._name is not None, "\"name\" field is required and cannot be empty" + assert self._type is not None, "\"type\" field is required and cannot be empty" + assert self._provider is not None, "\"provider\" field is required and cannot be empty" diff --git a/clients/client-python/gravitino/dto/requests/catalog_update_request.py b/clients/client-python/gravitino/dto/requests/catalog_update_request.py index 668c9410f17..e0edb652081 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_update_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_update_request.py @@ -14,10 +14,10 @@ @dataclass class CatalogUpdateRequestBase(RESTRequest): - type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name='@type')) def __init__(self, type: str): - self.type = type + self._type = type @abstractmethod def catalog_change(self): diff --git a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py index d61c31d5aa7..bbaab25352d 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_updates_request.py @@ -2,21 +2,22 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, List -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config from gravitino.dto.requests.catalog_update_request import CatalogUpdateRequest +from gravitino.rest.rest_message import RESTRequest @dataclass -class CatalogUpdatesRequest(DataClassJsonMixin): +class CatalogUpdatesRequest(RESTRequest): """Represents a request containing multiple catalog updates.""" - updates: Optional[List[CatalogUpdateRequest]] + _updates: Optional[List[CatalogUpdateRequest]] = field(metadata=config(field_name='updates'), default_factory=list) def __init__(self, updates: List[CatalogUpdateRequest] = None): - self.updates = updates + self._updates = updates def validate(self): """Validates each request in the list. @@ -24,8 +25,8 @@ def validate(self): Raises: IllegalArgumentException if validation of any request fails. """ - if self.updates is not None: - for update_request in self.updates: + if self._updates is not None: + for update_request in self._updates: update_request.validate() else: raise ValueError("Updates cannot be null") diff --git a/clients/client-python/gravitino/dto/requests/fileset_create_request.py b/clients/client-python/gravitino/dto/requests/fileset_create_request.py index a21ffe78929..0111f87b2e5 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_create_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_create_request.py @@ -5,19 +5,28 @@ from dataclasses import dataclass, field from typing import Optional, Dict -from dataclasses_json import DataClassJsonMixin, config +from dataclasses_json import config from gravitino.api.fileset import Fileset +from gravitino.rest.rest_message import RESTRequest @dataclass -class FilesetCreateRequest(DataClassJsonMixin): +class FilesetCreateRequest(RESTRequest): """Represents a request to create a fileset.""" - name: str - comment: Optional[str] - type: Fileset.Type - storage_location: str = field(metadata=config(field_name='storageLocation')) - properties: Dict[str, str] + _name: str = field(metadata=config(field_name='name')) + _comment: Optional[str] = field(metadata=config(field_name='comment')) + _type: Optional[Fileset.Type] = field(metadata=config(field_name='type')) + _storage_location: Optional[str] = field(metadata=config(field_name='storageLocation')) + _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) + + def __init__(self, name: str, comment: Optional[str] = None, type: Fileset.Type = None, + storage_location: Optional[str] = None, properties: Optional[Dict[str, str]] = None): + self._name = name + self._comment = comment + self._type = type + self._storage_location = storage_location + self._properties = properties def validate(self): """Validates the request. @@ -25,5 +34,5 @@ def validate(self): Raises: IllegalArgumentException if the request is invalid. """ - if not self.name: + if not self._name: raise ValueError('"name" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/dto/requests/fileset_update_request.py b/clients/client-python/gravitino/dto/requests/fileset_update_request.py index 4e294cae40f..ed4ed40a3b2 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_update_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_update_request.py @@ -30,12 +30,13 @@ class FilesetUpdateRequest: class RenameFilesetRequest(FilesetUpdateRequestBase): """The fileset update request for renaming a fileset.""" - new_name: str = field(metadata=config(field_name='newName')) + _new_name: str = field(metadata=config(field_name='newName')) """The new name for the Fileset.""" + def __init__(self, new_name: str): super().__init__("rename") - self.new_name = new_name + self._new_name = new_name def validate(self): """Validates the fields of the request. @@ -43,7 +44,7 @@ def validate(self): Raises: IllegalArgumentException if the new name is not set. """ - if not self.new_name: + if not self._new_name: raise ValueError('"new_name" field is required and cannot be empty') def fileset_change(self): @@ -52,18 +53,18 @@ def fileset_change(self): Returns: the fileset change. """ - return FilesetChange.rename(self.new_name) + return FilesetChange.rename(self._new_name) @dataclass class UpdateFilesetCommentRequest(FilesetUpdateRequestBase): """Represents a request to update the comment on a Fileset.""" - new_comment: str = field(metadata=config(field_name='newComment')) + _new_comment: str = field(metadata=config(field_name='newComment')) """The new comment for the Fileset.""" def __init__(self, new_comment: str): super().__init__("updateComment") - self.new_comment = new_comment + self._new_comment = new_comment def validate(self): """Validates the fields of the request. @@ -71,27 +72,27 @@ def validate(self): Raises: IllegalArgumentException if the new comment is not set. """ - if not self.new_comment: + if not self._new_comment: raise ValueError('"new_comment" field is required and cannot be empty') def fileset_change(self): """Returns the fileset change""" - return FilesetChange.update_comment(self.new_comment) + return FilesetChange.update_comment(self._new_comment) @dataclass class SetFilesetPropertyRequest(FilesetUpdateRequestBase): """Represents a request to set a property on a Fileset.""" - property: str = None + _property: str = field(metadata=config(field_name='property')) """The property to set.""" - value: str = None + _value: str = field(metadata=config(field_name='value')) """The value of the property.""" def __init__(self, property: str, value: str): super().__init__("setProperty") - self.property = property - self.value = value + self._property = property + self._value = value def validate(self): """Validates the fields of the request. @@ -99,24 +100,24 @@ def validate(self): Raises: IllegalArgumentException if property or value are not set. """ - if not self.property: + if not self._property: raise ValueError('"property" field is required and cannot be empty') - if not self.value: + if not self._value: raise ValueError('"value" field is required and cannot be empty') def fileset_change(self): - return FilesetChange.set_property(self.property, self.value) + return FilesetChange.set_property(self._property, self._value) @dataclass class RemoveFilesetPropertyRequest(FilesetUpdateRequestBase): """Represents a request to remove a property from a Fileset.""" - property: str = None + _property: str = field(metadata=config(field_name='property')) """The property to remove.""" def __init__(self, property: str): super().__init__("removeProperty") - self.property = property + self._property = property def validate(self): """Validates the fields of the request. @@ -124,8 +125,8 @@ def validate(self): Raises: IllegalArgumentException if property is not set. """ - if not self.property: + if not self._property: raise ValueError('"property" field is required and cannot be empty') def fileset_change(self): - return FilesetChange.remove_property(self.property) + return FilesetChange.remove_property(self._property) diff --git a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py index 12ba1dadca2..2552c1869f1 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_updates_request.py @@ -5,6 +5,8 @@ from dataclasses import dataclass, field from typing import Optional, List +from dataclasses_json import config + from gravitino.dto.requests.fileset_update_request import FilesetUpdateRequest from gravitino.rest.rest_message import RESTRequest @@ -12,10 +14,10 @@ @dataclass class FilesetUpdatesRequest(RESTRequest): """Request to represent updates to a fileset.""" - updates: List[FilesetUpdateRequest] = field(default_factory=list) + _updates: List[FilesetUpdateRequest] = field(metadata=config(field_name='updates'), default_factory=list) def validate(self): - if not self.updates: + if not self._updates: raise ValueError("Updates cannot be empty") - for update_request in self.updates: + for update_request in self._updates: update_request.validate() \ No newline at end of file diff --git a/clients/client-python/gravitino/dto/requests/metalake_create_request.py b/clients/client-python/gravitino/dto/requests/metalake_create_request.py index 080cdc8d0fb..4d89c57b7d5 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_create_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_create_request.py @@ -4,25 +4,31 @@ """ from typing import Optional, Dict -from dataclasses import dataclass +from dataclasses import dataclass, field -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config + +from gravitino.rest.rest_message import RESTRequest @dataclass -class MetalakeCreateRequest(DataClassJsonMixin): +class MetalakeCreateRequest(RESTRequest): """"Represents a request to create a Metalake.""" - name: str - comment: Optional[str] - properties: Optional[Dict[str, str]] + + _name: str = field(metadata=config(field_name='name')) + _comment: Optional[str] = field(metadata=config(field_name='comment')) + _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None): super().__init__() - self.name = name.strip() if name else None - self.comment = comment.strip() if comment else None - self.properties = properties + self._name = name.strip() if name else None + self._comment = comment.strip() if comment else None + self._properties = properties + + def name(self) -> str: + return self._name def validate(self): - if not self.name: + if not self._name: raise ValueError("\"name\" field is required and cannot be empty") diff --git a/clients/client-python/gravitino/dto/requests/metalake_update_request.py b/clients/client-python/gravitino/dto/requests/metalake_update_request.py index dfa639b6cff..1261b988c7e 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_update_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_update_request.py @@ -13,10 +13,10 @@ @dataclass class MetalakeUpdateRequestBase(RESTRequest): - type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name='@type')) def __init__(self, type: str): - self.type = type + self._type = type @abstractmethod def metalake_change(self): @@ -30,12 +30,12 @@ class MetalakeUpdateRequest: class RenameMetalakeRequest(MetalakeUpdateRequestBase): """Represents a request to rename a Metalake.""" - new_name: str = field(metadata=config(field_name='newName')) + _new_name: str = field(metadata=config(field_name='newName')) """The new name for the Metalake.""" def __init__(self, new_name: str): super().__init__("rename") - self.new_name = new_name + self._new_name = new_name def validate(self): """Validates the fields of the request. @@ -43,22 +43,22 @@ def validate(self): Raises: IllegalArgumentException if the new name is not set. """ - if not self.new_name: + if not self._new_name: raise ValueError('"newName" field is required and cannot be empty') def metalake_change(self): - return MetalakeChange.rename(self.new_name) + return MetalakeChange.rename(self._new_name) @dataclass class UpdateMetalakeCommentRequest(MetalakeUpdateRequestBase): """Represents a request to update the comment on a Metalake.""" - new_comment: str = field(metadata=config(field_name='newComment')) + _new_comment: str = field(metadata=config(field_name='newComment')) """The new comment for the Metalake.""" def __init__(self, new_comment: str): super().__init__("updateComment") - self.new_comment = new_comment + self._new_comment = new_comment def validate(self): """Validates the fields of the request. @@ -66,26 +66,26 @@ def validate(self): Raises: IllegalArgumentException if the new comment is not set. """ - if not self.new_comment: + if not self._new_comment: raise ValueError('"newComment" field is required and cannot be empty') def metalake_change(self): - return MetalakeChange.update_comment(self.new_comment) + return MetalakeChange.update_comment(self._new_comment) @dataclass class SetMetalakePropertyRequest(MetalakeUpdateRequestBase): """Represents a request to set a property on a Metalake.""" - property: str = None + _property: str = field(metadata=config(field_name='property')) """The property to set.""" - value: str = None + _value: str = field(metadata=config(field_name='value')) """The value of the property.""" def __init__(self, property: str, value: str): super().__init__("setProperty") - self.property = property - self.value = value + self._property = property + self._value = value def validate(self): """Validates the fields of the request. @@ -93,24 +93,24 @@ def validate(self): Raises: IllegalArgumentException if property or value are not set. """ - if not self.property: + if not self._property: raise ValueError('"property" field is required and cannot be empty') - if not self.value: + if not self._value: raise ValueError('"value" field is required and cannot be empty') def metalake_change(self): - return MetalakeChange.set_property(self.property, self.value) + return MetalakeChange.set_property(self._property, self._value) @dataclass class RemoveMetalakePropertyRequest(MetalakeUpdateRequestBase): """Represents a request to remove a property from a Metalake.""" - property: str = None + _property: str = field(metadata=config(field_name='property')) """The property to remove.""" def __init__(self, property: str): super().__init__("removeProperty") - self.property = property + self._property = property def validate(self): """Validates the fields of the request. @@ -118,8 +118,8 @@ def validate(self): Raises: IllegalArgumentException if property is not set. """ - if not self.property: + if not self._property: raise ValueError('"property" field is required and cannot be empty') def metalake_change(self): - return MetalakeChange.remove_property(self.property) + return MetalakeChange.remove_property(self._property) diff --git a/clients/client-python/gravitino/dto/requests/metalake_updates_request.py b/clients/client-python/gravitino/dto/requests/metalake_updates_request.py index 11ef2e1e8f3..3e30ce30150 100644 --- a/clients/client-python/gravitino/dto/requests/metalake_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/metalake_updates_request.py @@ -2,19 +2,20 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config from gravitino.dto.requests.metalake_update_request import MetalakeUpdateRequest +from gravitino.rest.rest_message import RESTRequest @dataclass -class MetalakeUpdatesRequest(DataClassJsonMixin): +class MetalakeUpdatesRequest(RESTRequest): """Represents a request containing multiple Metalake updates.""" - updates: List[MetalakeUpdateRequest] + _updates: List[MetalakeUpdateRequest] = field(metadata=config(field_name='updates')) def __init__(self, updates: List[MetalakeUpdateRequest]): """Constructor for MetalakeUpdatesRequest. @@ -22,7 +23,7 @@ def __init__(self, updates: List[MetalakeUpdateRequest]): Args: updates: The list of Metalake update requests. """ - self.updates = updates + self._updates = updates def validate(self): """Validates each request in the list. @@ -30,5 +31,5 @@ def validate(self): Raises: IllegalArgumentException if validation of any request fails. """ - for update in self.updates: + for update in self._updates: update.validate() diff --git a/clients/client-python/gravitino/dto/requests/schema_create_request.py b/clients/client-python/gravitino/dto/requests/schema_create_request.py index d77c668ab94..abaf78db6ab 100644 --- a/clients/client-python/gravitino/dto/requests/schema_create_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_create_request.py @@ -2,9 +2,11 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Dict +from dataclasses_json import config + from gravitino.rest.rest_message import RESTRequest @@ -12,9 +14,14 @@ class SchemaCreateRequest(RESTRequest): """Represents a request to create a schema.""" - name: str - comment: Optional[str] - properties: Optional[Dict[str, str]] + _name: str = field(metadata=config(field_name='name')) + _comment: Optional[str] = field(metadata=config(field_name='comment')) + _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) + + def __init__(self, name: str, comment: Optional[str], properties: Optional[Dict[str, str]]): + self._name = name + self._comment = comment + self._properties = properties def validate(self): - assert self.name is not None, "\"name\" field is required and cannot be empty" + assert self._name is not None, "\"name\" field is required and cannot be empty" diff --git a/clients/client-python/gravitino/dto/requests/schema_update_request.py b/clients/client-python/gravitino/dto/requests/schema_update_request.py index b11c7929273..83e4152b9fb 100644 --- a/clients/client-python/gravitino/dto/requests/schema_update_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_update_request.py @@ -13,15 +13,16 @@ @dataclass class SchemaUpdateRequestBase(RESTRequest): - type: str = field(metadata=config(field_name='@type')) + _type: str = field(metadata=config(field_name='@type')) def __init__(self, type: str): - self.type = type + self._type = type @abstractmethod def schema_change(self): pass + @dataclass class SchemaUpdateRequest: """Represents an interface for Schema update requests.""" @@ -30,16 +31,16 @@ class SchemaUpdateRequest: class SetSchemaPropertyRequest(SchemaUpdateRequestBase): """Represents a request to set a property on a Schema.""" - property: str = None + _property: str = field(metadata=config(field_name='property')) """The property to set.""" - value: str = None + _value: str = field(metadata=config(field_name='value')) """The value of the property.""" def __init__(self, property: str, value: str): super().__init__("setProperty") - self.property = property - self.value = value + self._property = property + self._value = value def validate(self): """Validates the fields of the request. @@ -47,24 +48,24 @@ def validate(self): Raises: IllegalArgumentException if property or value are not set. """ - if not self.property: + if not self._property: raise ValueError('"property" field is required and cannot be empty') - if not self.value: + if not self._value: raise ValueError('"value" field is required and cannot be empty') def schema_change(self): - return SchemaChange.set_property(self.property, self.value) + return SchemaChange.set_property(self._property, self._value) @dataclass class RemoveSchemaPropertyRequest(SchemaUpdateRequestBase): """Represents a request to remove a property from a Schema.""" - property: str = None + _property: str = field(metadata=config(field_name='property')) """The property to remove.""" def __init__(self, property: str): super().__init__("removeProperty") - self.property = property + self._property = property def validate(self): """Validates the fields of the request. @@ -72,8 +73,8 @@ def validate(self): Raises: IllegalArgumentException if property is not set. """ - if not self.property: + if not self._property: raise ValueError('"property" field is required and cannot be empty') def schema_change(self): - return SchemaChange.remove_property(self.property) + return SchemaChange.remove_property(self._property) diff --git a/clients/client-python/gravitino/dto/requests/schema_updates_request.py b/clients/client-python/gravitino/dto/requests/schema_updates_request.py index 084a76753a4..27a8e5ff550 100644 --- a/clients/client-python/gravitino/dto/requests/schema_updates_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_updates_request.py @@ -3,17 +3,21 @@ This software is licensed under the Apache License version 2. """ from dataclasses import dataclass, field -from typing import Optional, List +from typing import List -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config from gravitino.dto.requests.schema_update_request import SchemaUpdateRequest +from gravitino.rest.rest_message import RESTRequest @dataclass -class SchemaUpdatesRequest(DataClassJsonMixin): +class SchemaUpdatesRequest(RESTRequest): """Represents a request to update a schema.""" - updates: Optional[List[SchemaUpdateRequest]] = field(default_factory=list) + _updates: List[SchemaUpdateRequest] = field(metadata=config(field_name='updates'), default_factory=list) + + def __init__(self, updates: List[SchemaUpdateRequest]): + self._updates = updates def validate(self): """Validates the request. @@ -21,7 +25,7 @@ def validate(self): Raises: IllegalArgumentException If the request is invalid, this exception is thrown. """ - if not self.updates: + if not self._updates: raise ValueError("Updates cannot be empty") - for update_request in self.updates: + for update_request in self._updates: update_request.validate() \ No newline at end of file diff --git a/clients/client-python/gravitino/dto/responses/base_response.py b/clients/client-python/gravitino/dto/responses/base_response.py index a2eae0e398b..7c6644d267c 100644 --- a/clients/client-python/gravitino/dto/responses/base_response.py +++ b/clients/client-python/gravitino/dto/responses/base_response.py @@ -2,7 +2,9 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field + +from dataclasses_json import config from gravitino.rest.rest_message import RESTResponse @@ -11,11 +13,14 @@ class BaseResponse(RESTResponse): """Represents a base response for REST API calls.""" - code: int + _code: int = field(metadata=config(field_name='code')) + + def code(self) -> int: + return self._code def validate(self): """Validates the response code. TODO: @throws IllegalArgumentException if code value is negative. """ - if self.code < 0: + if self._code < 0: raise ValueError("code must be >= 0") diff --git a/clients/client-python/gravitino/dto/responses/catalog_list_response.py b/clients/client-python/gravitino/dto/responses/catalog_list_response.py index 78b637bc8f3..c38cc04892d 100644 --- a/clients/client-python/gravitino/dto/responses/catalog_list_response.py +++ b/clients/client-python/gravitino/dto/responses/catalog_list_response.py @@ -16,9 +16,5 @@ class CatalogListResponse(BaseResponse): """Represents a response for a list of catalogs with their information.""" _catalogs: List[CatalogDTO] = field(metadata=config(field_name='catalogs')) - def __init__(self, catalogs: List[CatalogDTO]): - super().__init__(0) - self._catalogs = catalogs - def catalogs(self) -> List[CatalogDTO]: return self._catalogs diff --git a/clients/client-python/gravitino/dto/responses/entity_list_response.py b/clients/client-python/gravitino/dto/responses/entity_list_response.py index 489ab1568a3..91beae6d3b9 100644 --- a/clients/client-python/gravitino/dto/responses/entity_list_response.py +++ b/clients/client-python/gravitino/dto/responses/entity_list_response.py @@ -14,7 +14,10 @@ @dataclass class EntityListResponse(BaseResponse): """Represents a response containing a list of catalogs.""" - idents: Optional[List[NameIdentifier]] = field(metadata=config(field_name='identifiers')) + _idents: List[NameIdentifier] = field(metadata=config(field_name='identifiers')) + + def identifiers(self) -> List[NameIdentifier]: + return self._idents def validate(self): """Validates the response data. @@ -24,4 +27,4 @@ def validate(self): """ super().validate() - assert self.idents is not None, "identifiers must not be null" \ No newline at end of file + assert self._idents is not None, "identifiers must not be null" \ No newline at end of file diff --git a/clients/client-python/gravitino/dto/responses/fileset_response.py b/clients/client-python/gravitino/dto/responses/fileset_response.py index c925ffbe5a8..502d9f24852 100644 --- a/clients/client-python/gravitino/dto/responses/fileset_response.py +++ b/clients/client-python/gravitino/dto/responses/fileset_response.py @@ -2,7 +2,9 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field + +from dataclasses_json import config from gravitino.dto.fileset_dto import FilesetDTO from gravitino.dto.responses.base_response import BaseResponse @@ -11,7 +13,10 @@ @dataclass class FilesetResponse(BaseResponse): """Response for fileset creation.""" - fileset: FilesetDTO + _fileset: FilesetDTO = field(metadata=config(field_name='fileset')) + + def fileset(self) -> FilesetDTO: + return self._fileset def validate(self): """Validates the response data. @@ -20,7 +25,7 @@ def validate(self): IllegalArgumentException if catalog identifiers are not set. """ super().validate() - assert self.fileset is not None, "fileset must not be null" - assert self.fileset.name, "fileset 'name' must not be null and empty" - assert self.fileset.storage_location, "fileset 'storageLocation' must not be null and empty" - assert self.fileset.type is not None, "fileset 'type' must not be null and empty" + assert self._fileset is not None, "fileset must not be null" + assert self._fileset.name, "fileset 'name' must not be null and empty" + assert self._fileset.storage_location, "fileset 'storageLocation' must not be null and empty" + assert self._fileset.type is not None, "fileset 'type' must not be null and empty" diff --git a/clients/client-python/gravitino/dto/responses/metalake_list_response.py b/clients/client-python/gravitino/dto/responses/metalake_list_response.py index afa01520713..b8679a3d521 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_list_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_list_response.py @@ -2,9 +2,11 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List +from dataclasses_json import config + from gravitino.dto.metalake_dto import MetalakeDTO from gravitino.dto.responses.base_response import BaseResponse @@ -13,7 +15,10 @@ class MetalakeListResponse(BaseResponse): """Represents a response containing a list of metalakes.""" - metalakes: List[MetalakeDTO] + _metalakes: List[MetalakeDTO] = field(metadata=config(field_name='metalakes')) + + def metalakes(self) -> List[MetalakeDTO]: + return self._metalakes def validate(self): """Validates the response data. @@ -23,11 +28,11 @@ def validate(self): """ super().validate() - if self.metalakes is None: + if self._metalakes is None: raise ValueError("metalakes must be non-null") - for metalake in self.metalakes: - if not metalake.name: + for metalake in self._metalakes: + if not metalake.name(): raise ValueError("metalake 'name' must not be null and empty") - if not metalake.audit: + if not metalake.audit_info(): raise ValueError("metalake 'audit' must not be null") diff --git a/clients/client-python/gravitino/dto/responses/metalake_response.py b/clients/client-python/gravitino/dto/responses/metalake_response.py index 0c238048787..c019029c9a3 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_response.py @@ -2,9 +2,11 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional +from dataclasses_json import config + from gravitino.dto.metalake_dto import MetalakeDTO from gravitino.dto.responses.base_response import BaseResponse @@ -13,7 +15,10 @@ class MetalakeResponse(BaseResponse): """Represents a response containing metalake information.""" - metalake: Optional[MetalakeDTO] + _metalake: Optional[MetalakeDTO] = field(metadata=config(field_name='metalake')) + + def metalake(self) -> MetalakeDTO: + return self._metalake def validate(self): """Validates the response data. @@ -21,6 +26,6 @@ def validate(self): """ super().validate() - assert self.metalake is not None, "metalake must not be null" - assert self.metalake.name is not None, "metalake 'name' must not be null and empty" - assert self.metalake.audit is not None, "metalake 'audit' must not be null" + assert self._metalake is not None, "metalake must not be null" + assert self._metalake.name() is not None, "metalake 'name' must not be null and empty" + assert self._metalake.audit_info() is not None, "metalake 'audit' must not be null" diff --git a/clients/client-python/gravitino/dto/responses/schema_response.py b/clients/client-python/gravitino/dto/responses/schema_response.py index f825c39bcce..04e497feed9 100644 --- a/clients/client-python/gravitino/dto/responses/schema_response.py +++ b/clients/client-python/gravitino/dto/responses/schema_response.py @@ -2,10 +2,9 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import DataClassJsonMixin, config from gravitino.dto.responses.base_response import BaseResponse from gravitino.dto.schema_dto import SchemaDTO @@ -14,7 +13,10 @@ @dataclass class SchemaResponse(BaseResponse, DataClassJsonMixin): """Represents a response for a schema.""" - schema: Optional[SchemaDTO] + _schema: SchemaDTO = field(metadata=config(field_name='schema')) + + def schema(self) -> SchemaDTO: + return self._schema def validate(self): """Validates the response data. @@ -24,6 +26,6 @@ def validate(self): """ super().validate() - assert self.schema is not None, "schema must be non-null" - assert self.schema.name is not None, "schema 'name' must not be null and empty" - assert self.schema.audit is not None, "schema 'audit' must not be null" + assert self._schema is not None, "schema must be non-null" + assert self._schema.name() is not None, "schema 'name' must not be null and empty" + assert self._schema.audit_info() is not None, "schema 'audit' must not be null" diff --git a/clients/client-python/gravitino/dto/schema_dto.py b/clients/client-python/gravitino/dto/schema_dto.py index f1638153a1c..04efe536a8a 100644 --- a/clients/client-python/gravitino/dto/schema_dto.py +++ b/clients/client-python/gravitino/dto/schema_dto.py @@ -2,33 +2,42 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Dict -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config +from gravitino.api.audit import Audit +from gravitino.api.schema import Schema from gravitino.dto.audit_dto import AuditDTO @dataclass -class SchemaDTO(DataClassJsonMixin): +class SchemaDTO(Schema): """Represents a Schema DTO (Data Transfer Object).""" - name: str + _name: str = field(metadata=config(field_name='name')) """The name of the Metalake DTO.""" - comment: Optional[str] + _comment: Optional[str] = field(metadata=config(field_name='comment')) """The comment of the Metalake DTO.""" - properties: Optional[Dict[str, str]] = None + _properties: Optional[Dict[str, str]] = field(metadata=config(field_name='properties')) """The properties of the Metalake DTO.""" - audit: AuditDTO = None + _audit: AuditDTO = field(metadata=config(field_name='audit')) """The audit information of the Metalake DTO.""" - def __init__(self, name: str = None, comment: str = None, properties: Dict[str, str] = None, - audit: AuditDTO = None): - self.name = name - self.comment = comment - self.properties = properties - self.audit = audit + def name(self) -> str: + return self._name + + def audit_info(self) -> Audit: + return self._audit + + def comment(self) -> Optional[str]: + """Returns the comment of the Schema. None is returned if the comment is not set.""" + return self._comment + + def properties(self) -> Dict[str, str]: + """Returns the properties of the Schema. An empty dictionary is returned if no properties are set.""" + return self._properties diff --git a/clients/client-python/gravitino/name_identifier.py b/clients/client-python/gravitino/name_identifier.py index d40ebd2cd67..f18805f5b15 100644 --- a/clients/client-python/gravitino/name_identifier.py +++ b/clients/client-python/gravitino/name_identifier.py @@ -2,25 +2,30 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ +from dataclasses import dataclass, field + +from dataclasses_json import DataClassJsonMixin, config + from gravitino.exceptions.illegal_name_identifier_exception import IllegalNameIdentifierException from gravitino.namespace import Namespace -class NameIdentifier: +@dataclass +class NameIdentifier(DataClassJsonMixin): """A name identifier is a sequence of names separated by dots. It's used to identify a metalake, a catalog, a schema or a table. For example, "metalake1" can represent a metalake, "metalake1.catalog1" can represent a catalog, "metalake1.catalog1.schema1" can represent a schema. """ - DOT: str = '.' + _name: str = field(metadata=config(field_name='name')) + _namespace: Namespace = field(metadata=config(field_name='namespace')) - _namespace: Namespace = None - _name: str = None + DOT: str = '.' - def __init__(self, namespace: Namespace, name: str): - self._namespace = namespace - self._name = name + @classmethod + def builder(cls, namespace: Namespace, name: str): + return NameIdentifier(_namespace=namespace, _name=name) def namespace(self): return self._namespace @@ -42,7 +47,7 @@ def of(*names: str) -> 'NameIdentifier': NameIdentifier.check(names is not None, "Cannot create a NameIdentifier with null names") NameIdentifier.check(len(names) > 0, "Cannot create a NameIdentifier with no names") - return NameIdentifier(Namespace.of(*names[:-1]), names[-1]) + return NameIdentifier.builder(Namespace.of(*names[:-1]), names[-1]) @staticmethod def of_namespace(namespace: Namespace, name: str) -> 'NameIdentifier': @@ -55,7 +60,7 @@ def of_namespace(namespace: Namespace, name: str) -> 'NameIdentifier': Returns: The created NameIdentifier """ - return NameIdentifier(namespace, name) + return NameIdentifier.builder(namespace, name) @staticmethod def of_metalake(metalake: str) -> 'NameIdentifier': @@ -237,7 +242,7 @@ def get_namespace(self): Returns: The namespace of the NameIdentifier. """ - return self.namespace + return self._namespace def get_name(self): """Get the name of the NameIdentifier. @@ -245,21 +250,21 @@ def get_name(self): Returns: The name of the NameIdentifier. """ - return self.name + return self._name def __eq__(self, other): if not isinstance(other, NameIdentifier): return False - return self.namespace == other.namespace and self.name == other.name + return self._namespace == other._namespace and self._name == other._name def __hash__(self): - return hash((self.namespace, self.name)) + return hash(self._namespace, self._name) def __str__(self): if self.has_namespace(): - return str(self.namespace) + "." + self.name + return str(self._namespace) + "." + self._name else: - return self.name + return self._name @staticmethod def check(condition, message, *args): diff --git a/clients/client-python/gravitino/namespace.py b/clients/client-python/gravitino/namespace.py index 281439f1cda..f4df5fbe31b 100644 --- a/clients/client-python/gravitino/namespace.py +++ b/clients/client-python/gravitino/namespace.py @@ -234,7 +234,7 @@ def __hash__(self) -> int: return hash(tuple(self._levels)) def __str__(self) -> str: - return Namespace._DOT.join(self._levels) + return self._DOT.join(self._levels) @staticmethod def check(expression: bool, message: str, *args) -> None: diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index e02206bf07c..ef4d7bbdaf0 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -38,21 +38,12 @@ def check_gravitino_server_status() -> bool: return gravitino_server_running -def _init_logging(): - logging.basicConfig(level=logging.DEBUG) - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) - logger.addHandler(console_handler) - - class IntegrationTestEnv(unittest.TestCase): """Provide real test environment for the Gravitino Server""" gravitino_startup_script = None @classmethod def setUpClass(cls): - _init_logging() - if os.environ.get('START_EXTERNAL_GRAVITINO') is not None: """Maybe Gravitino server already startup by Gradle test command or developer manual startup.""" if not check_gravitino_server_status(): diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 240547928f5..394b951cb54 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -3,14 +3,13 @@ This software is licensed under the Apache License version 2. """ import logging -from random import random, randint +from random import randint +from typing import Dict, List -from gravitino.api.catalog import Catalog from gravitino.api.fileset import Fileset from gravitino.api.fileset_change import FilesetChange from gravitino.client.gravitino_admin_client import GravitinoAdminClient from gravitino.client.gravitino_client import GravitinoClient -from gravitino.client.gravitino_metalake import GravitinoMetalake from gravitino.dto.catalog_dto import CatalogDTO from gravitino.name_identifier import NameIdentifier from tests.integration.integration_test_env import IntegrationTestEnv @@ -19,110 +18,114 @@ class TestFilesetCatalog(IntegrationTestEnv): - catalog: Catalog = None - metalake: GravitinoMetalake = None - metalake_name: str = "testMetalake" + str(randint(1, 100)) - catalog_name: str = "testCatalog" + str(randint(1, 100)) - schema_name: str = "testSchema" + str(randint(1, 100)) - fileset_name: str = "testFileset1" + str(randint(1, 100)) - fileset_alter_name: str = "testFilesetAlter" + str(randint(1, 100)) - provider: str = "hadoop" + metalake_name: str = "TestFilesetCatalog-metalake" + str(randint(1, 10000)) + catalog_name: str = "catalog" + catalog_location_pcatarop: str = "location" # Fileset Catalog must set `location` + catalog_provider: str = "hadoop" + + schema_name: str = "schema" + + fileset_name: str = "fileset" + fileset_alter_name: str = fileset_name + "Alter" + fileset_comment: str = "fileset_comment" + + fileset_location: str = "/tmp/TestFilesetCatalog" + fileset_properties_key1: str = "fileset_properties_key1" + fileset_properties_value1: str = "fileset_properties_value1" + fileset_properties_key2: str = "fileset_properties_key2" + fileset_properties_value2: str = "fileset_properties_value2" + fileset_properties: Dict[str, str] = {fileset_properties_key1: fileset_properties_value1, + fileset_properties_key2: fileset_properties_value2} + fileset_new_name = fileset_name + "_new" metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) catalog_ident: NameIdentifier = NameIdentifier.of_catalog(metalake_name, catalog_name) schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name) fileset_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, fileset_name) - fileset_alter_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, - fileset_alter_name) + fileset_new_ident: NameIdentifier = NameIdentifier.of_fileset(metalake_name, catalog_name, schema_name, + fileset_new_name) - gravitino_admin_client: GravitinoAdminClient = None + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient(uri="http://localhost:8090") gravitino_client: GravitinoClient = None - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.clean_test_data() - - cls.gravitino_admin_client = GravitinoAdminClient(uri="http://localhost:8090") - cls.metalake = cls.gravitino_admin_client.create_metalake(ident=cls.metalake_ident, - comment="test comment", properties={}) - cls.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=cls.metalake_name) - - cls.catalog = cls.gravitino_client.create_catalog( - ident=cls.catalog_ident, - type=CatalogDTO.Type.FILESET, - provider=cls.provider, - comment="comment", - properties={"k1": "v1"} - ) - - cls.catalog.as_schemas().create_schema(ident=cls.schema_ident, comment="comment", properties={"k1": "v1"}) + def setUp(self): + self.init_test_env() - @classmethod - def tearDownClass(cls): - """Clean test data""" - cls.clean_test_data() - super().tearDownClass() + def tearDown(self): + self.clean_test_data() - @classmethod - def clean_test_data(cls): + def clean_test_data(self): try: - cls.gravitino_admin_client = GravitinoAdminClient(uri="http://localhost:8090") - gravitino_metalake = cls.gravitino_admin_client.load_metalake(ident=cls.metalake_ident) - cls.catalog = gravitino_metalake.load_catalog(ident=cls.catalog_ident) - cls.catalog.as_fileset_catalog().drop_fileset(ident=cls.fileset_ident) - cls.catalog.as_fileset_catalog().drop_fileset(ident=cls.fileset_alter_ident) - cls.catalog.as_schemas().drop_schema(ident=cls.schema_ident, cascade=True) - gravitino_metalake.drop_catalog(ident=cls.catalog_ident) - cls.gravitino_admin_client.drop_metalake(cls.metalake_ident) + self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + logger.info("Drop fileset %s[%s]", self.fileset_ident, + catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident)) + logger.info("Drop fileset %s[%s]", self.fileset_new_ident, + catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_new_ident)) + logger.info("Drop schema %s[%s]", self.schema_ident, + catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True)) + logger.info("Drop catalog %s[%s]", self.catalog_ident, + self.gravitino_client.drop_catalog(ident=self.catalog_ident)) + logger.info("Drop metalake %s[%s]", self.metalake_ident, + self.gravitino_admin_client.drop_metalake(self.metalake_ident)) except Exception as e: - logger.debug(e) - - def create_catalog(self): - self.catalog = self.gravitino_client.create_catalog( - ident=self.catalog_ident, - type=CatalogDTO.Type.FILESET, - provider=self.provider, - comment="comment", - properties={"k1": "v1"}) - - assert self.catalog.name == self.catalog_name - assert self.catalog.type == CatalogDTO.Type.FILESET - assert self.catalog.provider == self.provider - - def create_schema(self): - self.catalog.as_schemas().create_schema( - ident=self.schema_ident, - comment="comment", - properties={"k1": "v1"}) + logger.error("Clean test data failed: %s", e) + + def init_test_env(self): + self.gravitino_admin_client.create_metalake(ident=self.metalake_ident, comment="", properties={}) + self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) + catalog = self.gravitino_client.create_catalog(ident=self.catalog_ident, + type=CatalogDTO.Type.FILESET, + provider=self.catalog_provider, comment="", + properties={self.catalog_location_pcatarop: "/tmp/test1"}) + catalog.as_schemas().create_schema(ident=self.schema_ident, comment="", properties={}) + + def create_fileset(self) -> Fileset: + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + return (catalog.as_fileset_catalog().create_fileset(ident=self.fileset_ident, + type=Fileset.Type.MANAGED, + comment=self.fileset_comment, + storage_location=self.fileset_location, + properties=self.fileset_properties)) def test_create_fileset(self): - fileset = self.catalog.as_fileset_catalog().create_fileset(ident=self.fileset_ident, - type=Fileset.Type.MANAGED, - comment="mock comment", - storage_location="mock location", - properties={"k1": "v1"}) - assert fileset is not None - - fileset_list = self.catalog.as_fileset_catalog().list_filesets(self.fileset_ident.namespace()) - assert fileset_list is not None and len(fileset_list) == 1 - - fileset = self.catalog.as_fileset_catalog().load_fileset(self.fileset_ident) - assert fileset is not None - assert fileset.name() == self.fileset_ident.name() + fileset = self.create_fileset() + self.assertIsNotNone(fileset) + self.assertEqual(fileset.type(), Fileset.Type.MANAGED) + self.assertEqual(fileset.comment(), self.fileset_comment) + self.assertEqual(fileset.properties(), self.fileset_properties) + + def test_drop_fileset(self): + self.create_fileset() + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + self.assertTrue(catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident)) + + def test_list_fileset(self): + self.create_fileset() + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + fileset_list: List[NameIdentifier] = (catalog.as_fileset_catalog() + .list_filesets(namespace=self.fileset_ident.namespace())) + self.assertTrue(any(item.name() == self.fileset_name for item in fileset_list)) + + def test_load_fileset(self): + self.create_fileset() + fileset = (self.gravitino_client.load_catalog(ident=self.catalog_ident) + .as_fileset_catalog().load_fileset(ident=self.fileset_ident)) + self.assertIsNotNone(fileset) + self.assertEqual(fileset.name(), self.fileset_name) + self.assertEqual(fileset.comment(), self.fileset_comment) + self.assertEqual(fileset.properties(), self.fileset_properties) + self.assertEqual(fileset.audit_info().creator(), "anonymous") + + def test_alter_fileset(self): + self.create_fileset() + fileset_propertie_new_value = self.fileset_properties_value2 + "_new" - # Alter fileset changes = ( - FilesetChange.rename(self.fileset_alter_name), - FilesetChange.update_comment("new fileset comment"), - FilesetChange.set_property("key1", "value1"), - FilesetChange.remove_property("k1"), + FilesetChange.remove_property(self.fileset_properties_key1), + FilesetChange.set_property(self.fileset_properties_key2, fileset_propertie_new_value), ) - fileset_alter = self.catalog.as_fileset_catalog().alter_fileset(self.fileset_ident, *changes) - assert fileset_alter is not None - assert fileset_alter.name() == self.fileset_alter_name - assert fileset_alter.comment() == "new fileset comment" - assert fileset_alter.properties().get("key1") == "value1" - - # Clean test data - self.catalog.as_fileset_catalog().drop_fileset(ident=self.fileset_ident) + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + fileset_new = catalog.as_fileset_catalog().alter_fileset(self.fileset_ident, *changes) + self.assertEqual(fileset_new.properties().get(self.fileset_properties_key2), fileset_propertie_new_value) + self.assertTrue(self.fileset_properties_key1 not in fileset_new.properties()) diff --git a/clients/client-python/tests/integration/test_gravitino_admin_client.py b/clients/client-python/tests/integration/test_gravitino_admin_client.py deleted file mode 100644 index e54f4a1ac13..00000000000 --- a/clients/client-python/tests/integration/test_gravitino_admin_client.py +++ /dev/null @@ -1,116 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" -import logging - -import gravitino -from gravitino.client.gravitino_admin_client import GravitinoAdminClient -from gravitino.dto.dto_converters import DTOConverters -from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest -from gravitino.dto.responses.metalake_response import MetalakeResponse -from gravitino.api.metalake_change import MetalakeChange -from gravitino.name_identifier import NameIdentifier -from gravitino.utils.exceptions import NotFoundError -from tests.integration.integration_test_env import IntegrationTestEnv - -logger = logging.getLogger(__name__) - - -class TestGravitinoAdminClient(IntegrationTestEnv): - def setUp(self): - self._gravitino_admin_client = GravitinoAdminClient(uri="http://localhost:8090") - - def test_create_metalake(self): - metalake_name = "metalake00" - - # Clean test data - self.drop_metalake(metalake_name) - - self.create_metalake(metalake_name) - # Clean test data - self.drop_metalake(metalake_name) - - def create_metalake(self, metalake_name): - ident = NameIdentifier.of(metalake_name) - comment = "This is a sample comment" - properties = {"key1": "value1", "key2": "value2"} - - gravitinoMetalake = self._gravitino_admin_client.create_metalake(ident, comment, properties) - - self.assertEqual(gravitinoMetalake.name, metalake_name) - self.assertEqual(gravitinoMetalake.comment, comment) - self.assertEqual(gravitinoMetalake.properties.get("key1"), "value1") - self.assertEqual(gravitinoMetalake.audit._creator, "anonymous") - - def test_alter_metalake(self): - metalake_name = "metalake02" - metalake_new_name = metalake_name + "_new" - - # Clean test data - self.drop_metalake(metalake_name) - self.drop_metalake(metalake_new_name) - - self.create_metalake(metalake_name) - changes = ( - MetalakeChange.rename(metalake_new_name), - MetalakeChange.update_comment("new metalake comment"), - ) - - metalake = self._gravitino_admin_client.alter_metalake(NameIdentifier.of(metalake_name), *changes) - self.assertEqual(metalake_new_name, metalake.name) - self.assertEqual("new metalake comment", metalake.comment) - self.assertEqual("anonymous", metalake.audit._creator) # Assuming a constant or similar attribute - - # Reload metadata via new name to check if the changes are applied - new_metalake = self._gravitino_admin_client.load_metalake(NameIdentifier.of(metalake_new_name)) - self.assertEqual(metalake_new_name, new_metalake.name) - self.assertEqual("new metalake comment", new_metalake.comment) - - # Old name does not exist - old = NameIdentifier.of(metalake_name) - with self.assertRaises(NotFoundError): # TODO: NoSuchMetalakeException - self._gravitino_admin_client.load_metalake(old) - - def drop_metalake(self, metalake_name) -> bool: - ident = NameIdentifier.of(metalake_name) - return self._gravitino_admin_client.drop_metalake(ident) - - def test_drop_metalake(self): - metalake_name = "metalake03" - try: - self.create_metalake(metalake_name) - except gravitino.utils.exceptions.HTTPError: - self.drop_metalake(metalake_name) - - assert self.drop_metalake(metalake_name) == True - - def test_metalake_update_request_to_json(self): - changes = ( - MetalakeChange.rename("my_metalake_new"), - MetalakeChange.update_comment("new metalake comment"), - ) - reqs = [DTOConverters.to_metalake_update_request(change) for change in changes] - updates_request = MetalakeUpdatesRequest(reqs) - valid_json = (f'{{"updates": [{{"@type": "rename", "newName": "my_metalake_new"}}, ' - f'{{"@type": "updateComment", "newComment": "new metalake comment"}}]}}') - self.assertEqual(updates_request.to_json(), valid_json) - - def test_from_json_metalake_response(self): - str = (b'{"code":0,"metalake":{"name":"example_name18","comment":"This is a sample comment",' - b'"properties":{"key1":"value1","key2":"value2"},' - b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}') - metalake_response = MetalakeResponse.from_json(str, infer_missing=True) - self.assertEqual(metalake_response.code, 0) - self.assertIsNotNone(metalake_response.metalake) - self.assertEqual(metalake_response.metalake.name, "example_name18") - self.assertEqual(metalake_response.metalake.audit._creator, "anonymous") - - def test_list_metalakes(self): - metalake_name = "metalake05" - self.create_metalake(metalake_name) - object = self._gravitino_admin_client.list_metalakes() - assert len(object) > 0 - - # Clean test data - self.drop_metalake(metalake_name) diff --git a/clients/client-python/tests/integration/test_metalake.py b/clients/client-python/tests/integration/test_metalake.py new file mode 100644 index 00000000000..612245d460d --- /dev/null +++ b/clients/client-python/tests/integration/test_metalake.py @@ -0,0 +1,125 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import logging +from typing import Dict, List + +from gravitino.client.gravitino_admin_client import GravitinoAdminClient +from gravitino.client.gravitino_metalake import GravitinoMetalake +from gravitino.dto.dto_converters import DTOConverters +from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest +from gravitino.dto.responses.metalake_response import MetalakeResponse +from gravitino.api.metalake_change import MetalakeChange +from gravitino.name_identifier import NameIdentifier +from tests.integration.integration_test_env import IntegrationTestEnv + +logger = logging.getLogger(__name__) + + +class TestMetalake(IntegrationTestEnv): + metalake_name: str = "TestMetalake-metalake" + metalake_new_name = metalake_name + "_new" + + metalake_comment: str = "metalake_comment" + metalake_properties_key1: str = "metalake_properties_key1" + metalake_properties_value1: str = "metalake_properties_value1" + metalake_properties_key2: str = "metalake_properties_key2" + metalake_properties_value2: str = "metalake_properties_value2" + metalake_properties: Dict[str, str] = {metalake_properties_key1: metalake_properties_value1, + metalake_properties_key2: metalake_properties_value2} + + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient(uri="http://localhost:8090") + + def tearDown(self): + self.clean_test_data() + + def clean_test_data(self): + logger.info("Drop metalake %s[%s]", self.metalake_name, self.drop_metalake(self.metalake_name)) + logger.info("Drop metalake %s[%s]", self.metalake_new_name, self.drop_metalake(self.metalake_new_name)) + + def test_create_metalake(self): + metalake = self.create_metalake(self.metalake_name) + self.assertEqual(metalake.name(), self.metalake_name) + self.assertEqual(metalake.comment(), self.metalake_comment) + self.assertEqual(metalake.properties(), self.metalake_properties) + self.assertEqual(metalake.audit_info().creator(), "anonymous") + + def create_metalake(self, metalake_name) -> GravitinoMetalake: + return self.gravitino_admin_client.create_metalake(NameIdentifier.of(metalake_name), + self.metalake_comment, + self.metalake_properties) + + def test_alter_metalake(self): + self.create_metalake(self.metalake_name) + + metalake_new_name = self.metalake_name + "_new" + metalake_new_comment = self.metalake_comment + "_new" + metalake_propertie_new_value: str = "metalake_propertie_new_value1" + + changes = ( + MetalakeChange.rename(metalake_new_name), + MetalakeChange.update_comment(metalake_new_comment), + MetalakeChange.remove_property(self.metalake_properties_key1), + MetalakeChange.set_property(self.metalake_properties_key2, metalake_propertie_new_value), + ) + + metalake = self.gravitino_admin_client.alter_metalake(NameIdentifier.of(self.metalake_name), *changes) + self.assertEqual(metalake.name(), metalake_new_name) + self.assertEqual(metalake.comment(), metalake_new_comment) + self.assertEqual(metalake.properties().get(self.metalake_properties_key2), metalake_propertie_new_value) + self.assertTrue(self.metalake_properties_key1 not in metalake.properties()) + + def drop_metalake(self, metalake_name: str) -> bool: + ident = NameIdentifier.of(metalake_name) + return self.gravitino_admin_client.drop_metalake(ident) + + def test_drop_metalake(self): + self.create_metalake(self.metalake_name) + self.assertTrue(self.drop_metalake(self.metalake_name)) + + def test_metalake_update_request_to_json(self): + changes = ( + MetalakeChange.rename("my_metalake_new"), + MetalakeChange.update_comment("new metalake comment"), + ) + reqs = [DTOConverters.to_metalake_update_request(change) for change in changes] + updates_request = MetalakeUpdatesRequest(reqs) + valid_json = (f'{{"updates": [{{"@type": "rename", "newName": "my_metalake_new"}}, ' + f'{{"@type": "updateComment", "newComment": "new metalake comment"}}]}}') + self.assertEqual(updates_request.to_json(), valid_json) + + def test_from_json_metalake_response(self): + str_json = (b'{"code":0,"metalake":{"name":"example_name18","comment":"This is a sample comment",' + b'"properties":{"key1":"value1","key2":"value2"},' + b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}') + metalake_response = MetalakeResponse.from_json(str_json, infer_missing=True) + self.assertEqual(metalake_response.code(), 0) + self.assertIsNotNone(metalake_response._metalake) + self.assertEqual(metalake_response._metalake.name(), "example_name18") + self.assertEqual(metalake_response._metalake.audit_info().creator(), "anonymous") + + def test_from_error_json_metalake_response(self): + str_json = (b'{"code":0, "undefine-key1":"undefine-value1", ' + b'"metalake":{"undefine-key2":1, "name":"example_name18","comment":"This is a sample comment",' + b'"properties":{"key1":"value1","key2":"value2"},' + b'"audit":{"creator":"anonymous","createTime":"2024-04-05T10:10:35.218Z"}}}') + metalake_response = MetalakeResponse.from_json(str_json, infer_missing=True) + self.assertEqual(metalake_response.code(), 0) + self.assertIsNotNone(metalake_response._metalake) + self.assertEqual(metalake_response._metalake.name(), "example_name18") + self.assertEqual(metalake_response._metalake.audit_info().creator(), "anonymous") + + def test_list_metalakes(self): + self.create_metalake(self.metalake_name) + metalake_list: List[GravitinoMetalake] = self.gravitino_admin_client.list_metalakes() + self.assertTrue(any(item.name() == self.metalake_name for item in metalake_list)) + + def test_load_metalakes(self): + self.create_metalake(self.metalake_name) + metalake = self.gravitino_admin_client.load_metalake(NameIdentifier.of(self.metalake_name)) + self.assertIsNotNone(metalake) + self.assertEqual(metalake.name(), self.metalake_name) + self.assertEqual(metalake.comment(), self.metalake_comment) + self.assertEqual(metalake.properties(), self.metalake_properties) + self.assertEqual(metalake.audit_info().creator(), "anonymous") diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py new file mode 100644 index 00000000000..a0fe41972dd --- /dev/null +++ b/clients/client-python/tests/integration/test_schema.py @@ -0,0 +1,120 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import logging +from random import randint +from typing import Dict, List + +from gravitino.api.catalog import Catalog +from gravitino.api.schema import Schema +from gravitino.api.schema_change import SchemaChange +from gravitino.client.gravitino_admin_client import GravitinoAdminClient +from gravitino.client.gravitino_client import GravitinoClient +from gravitino.name_identifier import NameIdentifier +from tests.integration.integration_test_env import IntegrationTestEnv + +logger = logging.getLogger(__name__) + + +class TestSchema(IntegrationTestEnv): + metalake_name: str = "TestSchema-metalake" + str(randint(1, 10000)) + + catalog_name: str = "testCatalog" + catalog_location_prop: str = "location" # Fileset Catalog must set `location` + catalog_provider: str = "hadoop" + + schema_name: str = "testSchema" + schema_new_name = schema_name + "_new" + + schema_comment: str = "schema_comment" + schema_properties_key1: str = "schema_properties_key1" + schema_properties_value1: str = "schema_properties_value1" + schema_properties_key2: str = "schema_properties_key2" + schema_properties_value2: str = "schema_properties_value2" + schema_properties: Dict[str, str] = {schema_properties_key1: schema_properties_value1, + schema_properties_key2: schema_properties_value2} + + metalake_ident: NameIdentifier = NameIdentifier.of(metalake_name) + catalog_ident: NameIdentifier = NameIdentifier.of_catalog(metalake_name, catalog_name) + schema_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_name) + schema_new_ident: NameIdentifier = NameIdentifier.of_schema(metalake_name, catalog_name, schema_new_name) + + gravitino_admin_client: GravitinoAdminClient = GravitinoAdminClient(uri="http://localhost:8090") + gravitino_client: GravitinoClient = None + + def setUp(self): + self.init_test_env() + + def tearDown(self): + self.clean_test_data() + + def init_test_env(self): + self.gravitino_admin_client.create_metalake(ident=self.metalake_ident, comment="", properties={}) + self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) + self.gravitino_client.create_catalog(ident=self.catalog_ident, type=Catalog.Type.FILESET, + provider=self.catalog_provider, comment="", + properties={self.catalog_location_prop: "/tmp/test_schema"}) + + def clean_test_data(self): + try: + self.gravitino_client = GravitinoClient(uri="http://localhost:8090", metalake_name=self.metalake_name) + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + logger.info("Drop schema %s[%s]", self.schema_ident, + catalog.as_schemas().drop_schema(self.schema_ident, cascade=True)) + logger.info("Drop schema %s[%s]", self.schema_new_ident, + catalog.as_schemas().drop_schema(self.schema_new_ident, cascade=True)) + logger.info("Drop catalog %s[%s]", self.catalog_ident, + self.gravitino_client.drop_catalog(ident=self.catalog_ident)) + logger.info("Drop metalake %s[%s]", self.metalake_ident, + self.gravitino_admin_client.drop_metalake(self.metalake_ident)) + except Exception as e: + logger.error("Clean test data failed: %s", e) + + def create_schema(self) -> Schema: + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + return catalog.as_schemas().create_schema(ident=self.schema_ident, + comment=self.schema_comment, + properties=self.schema_properties) + + def test_create_schema(self): + schema = self.create_schema() + self.assertEqual(schema.name(), self.schema_name) + self.assertEqual(schema.comment(), self.schema_comment) + self.assertEqual(schema.properties(), self.schema_properties) + self.assertEqual(schema.audit_info().creator(), "anonymous") + + def test_drop_schema(self): + self.create_schema() + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + self.assertTrue(catalog.as_schemas().drop_schema(ident=self.schema_ident, cascade=True)) + + def test_list_schema(self): + self.create_schema() + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + schema_list: List[NameIdentifier] = catalog.as_schemas().list_schemas( + namespace=self.schema_ident.namespace()) + self.assertTrue(any(item.name() == self.schema_name for item in schema_list)) + + def test_load_schema(self): + self.create_schema() + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + schema = catalog.as_schemas().load_schema(ident=self.schema_ident) + self.assertIsNotNone(schema) + self.assertEqual(schema.name(), self.schema_name) + self.assertEqual(schema.comment(), self.schema_comment) + self.assertEqual(schema.properties(), self.schema_properties) + self.assertEqual(schema.audit_info().creator(), "anonymous") + + def test_alter_schema(self): + self.create_schema() + schema_propertie_new_value = self.schema_properties_value2 + "_new" + + changes = ( + SchemaChange.remove_property(self.schema_properties_key1), + SchemaChange.set_property(self.schema_properties_key2, schema_propertie_new_value), + ) + catalog = self.gravitino_client.load_catalog(ident=self.catalog_ident) + schema_new = catalog.as_schemas().alter_schema(self.schema_ident, *changes) + self.assertEqual(schema_new.properties().get(self.schema_properties_key2), schema_propertie_new_value) + self.assertTrue(self.schema_properties_key1 not in schema_new.properties())