-
Notifications
You must be signed in to change notification settings - Fork 74
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: Use a single source of truth for built-in capabilities
1 parent
4674b3f
commit 93bbe08
Showing
19 changed files
with
709 additions
and
523 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
Built-in Settings and Capabilities | ||
================================== | ||
|
||
.. currentmodule:: singer_sdk.helpers.capabilities | ||
|
||
The Singer SDK library provides a number of built-in settings and capabilities. | ||
|
||
.. autodata:: ADD_RECORD_METADATA | ||
:no-value: | ||
|
||
.. autoattribute:: ADD_RECORD_METADATA.schema | ||
|
||
.. autodata:: BATCH | ||
:no-value: | ||
|
||
.. autoattribute:: BATCH.schema | ||
.. autoattribute:: BATCH.capability |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
"""Module with helpers to declare capabilities and plugin behavior.""" | ||
|
||
from __future__ import annotations | ||
|
||
from singer_sdk.helpers.capabilities import _schema as schema | ||
from singer_sdk.helpers.capabilities._builtin import Builtin | ||
from singer_sdk.helpers.capabilities._config_property import ConfigProperty | ||
from singer_sdk.helpers.capabilities._enum import ( | ||
CapabilitiesEnum, | ||
PluginCapabilities, | ||
TapCapabilities, | ||
TargetCapabilities, | ||
TargetLoadMethods, | ||
) | ||
|
||
__all__ = [ | ||
"ADD_RECORD_METADATA", | ||
"BATCH", | ||
"FLATTENING", | ||
"STREAM_MAPS", | ||
"TARGET_BATCH_SIZE_ROWS", | ||
"TARGET_HARD_DELETE", | ||
"TARGET_LOAD_METHOD", | ||
"TARGET_SCHEMA", | ||
"TARGET_VALIDATE_RECORDS", | ||
"CapabilitiesEnum", | ||
"ConfigProperty", | ||
"PluginCapabilities", | ||
"TapCapabilities", | ||
"TargetCapabilities", | ||
"TargetLoadMethods", | ||
] | ||
|
||
#: Add metadata to records. | ||
#: | ||
#: Example: | ||
#: | ||
#: .. code-block:: json | ||
#: | ||
#: { | ||
#: "add_record_metadata": true | ||
#: } | ||
#: | ||
ADD_RECORD_METADATA = Builtin(schema=schema.ADD_RECORD_METADATA_CONFIG) | ||
|
||
#: For taps, support emitting BATCH messages. For targets, support consuming BATCH | ||
#: messages. | ||
BATCH = Builtin( | ||
schema=schema.BATCH_CONFIG, | ||
capability=PluginCapabilities.BATCH, | ||
) | ||
|
||
FLATTENING = Builtin( | ||
schema=schema.FLATTENING_CONFIG, | ||
capability=PluginCapabilities.FLATTENING, | ||
) | ||
STREAM_MAPS = Builtin( | ||
schema.STREAM_MAPS_CONFIG, | ||
capability=PluginCapabilities.STREAM_MAPS, | ||
) | ||
TARGET_BATCH_SIZE_ROWS = Builtin(schema=schema.TARGET_BATCH_SIZE_ROWS_CONFIG) | ||
TARGET_HARD_DELETE = Builtin( | ||
schema=schema.TARGET_HARD_DELETE_CONFIG, | ||
capability=TargetCapabilities.HARD_DELETE, | ||
) | ||
TARGET_LOAD_METHOD = Builtin(schema=schema.TARGET_LOAD_METHOD_CONFIG) | ||
TARGET_SCHEMA = Builtin( | ||
schema=schema.TARGET_SCHEMA_CONFIG, | ||
capability=TargetCapabilities.TARGET_SCHEMA, | ||
) | ||
TARGET_VALIDATE_RECORDS = Builtin( | ||
schema=schema.TARGET_VALIDATE_RECORDS_CONFIG, | ||
capability=TargetCapabilities.VALIDATE_RECORDS, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
from __future__ import annotations | ||
|
||
import typing as t | ||
|
||
from ._config_property import ConfigProperty | ||
|
||
if t.TYPE_CHECKING: | ||
from ._enum import CapabilitiesEnum | ||
|
||
_T = t.TypeVar("_T") | ||
|
||
|
||
class Builtin: | ||
"""Use this class to define built-in setting(s) for a plugin.""" | ||
|
||
def __init__( | ||
self, | ||
schema: dict[str, t.Any], | ||
*, | ||
capability: CapabilitiesEnum | None = None, | ||
**kwargs: t.Any, | ||
): | ||
"""Initialize the descriptor. | ||
Args: | ||
schema: The JSON schema for the setting. | ||
capability: The capability that the setting is associated with. | ||
kwargs: Additional keyword arguments. | ||
""" | ||
self.schema = schema | ||
self.capability = capability | ||
self.kwargs = kwargs | ||
|
||
def attribute( # noqa: PLR6301 | ||
self, | ||
custom_key: str | None = None, | ||
*, | ||
default: _T | None = None, | ||
) -> ConfigProperty[_T]: | ||
"""Generate a class attribute for the setting. | ||
Args: | ||
custom_key: Custom key to use in the config. | ||
default: Default value for the setting. | ||
Returns: | ||
Class attribute for the setting. | ||
""" | ||
return ConfigProperty(custom_key=custom_key, default=default) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
from __future__ import annotations | ||
|
||
import typing as t | ||
|
||
T = t.TypeVar("T") | ||
|
||
|
||
class ConfigProperty(t.Generic[T]): | ||
"""A descriptor that gets a value from a named key of the config attribute.""" | ||
|
||
def __init__(self, custom_key: str | None = None, *, default: T | None = None): | ||
"""Initialize the descriptor. | ||
Args: | ||
custom_key: The key to get from the config attribute instead of the | ||
attribute name. | ||
default: The default value if the key is not found. | ||
""" | ||
self.key = custom_key | ||
self.default = default | ||
|
||
def __set_name__(self, owner, name: str) -> None: # noqa: ANN001 | ||
"""Set the name of the attribute. | ||
Args: | ||
owner: The class of the object. | ||
name: The name of the attribute. | ||
""" | ||
self.key = self.key or name | ||
|
||
def __get__(self, instance, owner) -> T | None: # noqa: ANN001 | ||
"""Get the value from the instance's config attribute. | ||
Args: | ||
instance: The instance of the object. | ||
owner: The class of the object. | ||
Returns: | ||
The value from the config attribute. | ||
""" | ||
return instance.config.get(self.key, self.default) # type: ignore[no-any-return] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,200 @@ | ||
from __future__ import annotations | ||
|
||
import enum | ||
import typing as t | ||
import warnings | ||
|
||
_EnumMemberT = t.TypeVar("_EnumMemberT") | ||
|
||
|
||
class TargetLoadMethods(str, enum.Enum): | ||
"""Target-specific capabilities.""" | ||
|
||
# always write all input records whether that records already exists or not | ||
APPEND_ONLY = "append-only" | ||
|
||
# update existing records and insert new records | ||
UPSERT = "upsert" | ||
|
||
# delete all existing records and insert all input records | ||
OVERWRITE = "overwrite" | ||
|
||
|
||
class DeprecatedEnum(enum.Enum): | ||
"""Base class for capabilities enumeration.""" | ||
|
||
def __new__( | ||
cls, | ||
value: _EnumMemberT, | ||
deprecation: str | None = None, | ||
) -> DeprecatedEnum: | ||
"""Create a new enum member. | ||
Args: | ||
value: Enum member value. | ||
deprecation: Deprecation message. | ||
Returns: | ||
An enum member value. | ||
""" | ||
member: DeprecatedEnum = object.__new__(cls) | ||
member._value_ = value | ||
member.deprecation = deprecation | ||
return member | ||
|
||
@property | ||
def deprecation_message(self) -> str | None: | ||
"""Get deprecation message. | ||
Returns: | ||
Deprecation message. | ||
""" | ||
self.deprecation: str | None | ||
return self.deprecation | ||
|
||
def emit_warning(self) -> None: | ||
"""Emit deprecation warning.""" | ||
warnings.warn( | ||
f"{self.name} is deprecated. {self.deprecation_message}", | ||
DeprecationWarning, | ||
stacklevel=3, | ||
) | ||
|
||
|
||
class DeprecatedEnumMeta(enum.EnumMeta): | ||
"""Metaclass for enumeration with deprecation support.""" | ||
|
||
def __getitem__(self, name: str) -> t.Any: # noqa: ANN401 | ||
"""Retrieve mapping item. | ||
Args: | ||
name: Item name. | ||
Returns: | ||
Enum member. | ||
""" | ||
obj: enum.Enum = super().__getitem__(name) | ||
if isinstance(obj, DeprecatedEnum) and obj.deprecation_message: | ||
obj.emit_warning() | ||
return obj | ||
|
||
def __getattribute__(cls, name: str) -> t.Any: # noqa: ANN401, N805 | ||
"""Retrieve enum attribute. | ||
Args: | ||
name: Attribute name. | ||
Returns: | ||
Attribute. | ||
""" | ||
obj = super().__getattribute__(name) | ||
if isinstance(obj, DeprecatedEnum) and obj.deprecation_message: | ||
obj.emit_warning() | ||
return obj | ||
|
||
def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: # noqa: ANN401 | ||
"""Call enum member. | ||
Args: | ||
args: Positional arguments. | ||
kwargs: Keyword arguments. | ||
Returns: | ||
Enum member. | ||
""" | ||
obj = super().__call__(*args, **kwargs) | ||
if isinstance(obj, DeprecatedEnum) and obj.deprecation_message: | ||
obj.emit_warning() | ||
return obj | ||
|
||
|
||
class CapabilitiesEnum(DeprecatedEnum, metaclass=DeprecatedEnumMeta): | ||
"""Base capabilities enumeration.""" | ||
|
||
def __str__(self) -> str: | ||
"""String representation. | ||
Returns: | ||
Stringified enum value. | ||
""" | ||
return str(self.value) | ||
|
||
def __repr__(self) -> str: | ||
"""String representation. | ||
Returns: | ||
Stringified enum value. | ||
""" | ||
return str(self.value) | ||
|
||
|
||
class PluginCapabilities(CapabilitiesEnum): | ||
"""Core capabilities which can be supported by taps and targets.""" | ||
|
||
#: Support plugin capability and setting discovery. | ||
ABOUT = "about" | ||
|
||
#: Support :doc:`inline stream map transforms</stream_maps>`. | ||
STREAM_MAPS = "stream-maps" | ||
|
||
#: Support schema flattening, aka de-nesting of complex properties. | ||
FLATTENING = "schema-flattening" | ||
|
||
#: Support the | ||
#: `ACTIVATE_VERSION <https://hub.meltano.com/singer/docs#activate-version>`_ | ||
#: extension. | ||
ACTIVATE_VERSION = "activate-version" | ||
|
||
#: Input and output from | ||
#: `batched files <https://hub.meltano.com/singer/docs#batch>`_. | ||
#: A.K.A ``FAST_SYNC``. | ||
BATCH = "batch" | ||
|
||
|
||
class TapCapabilities(CapabilitiesEnum): | ||
"""Tap-specific capabilities.""" | ||
|
||
#: Generate a catalog with `--discover`. | ||
DISCOVER = "discover" | ||
|
||
#: Accept input catalog, apply metadata and selection rules. | ||
CATALOG = "catalog" | ||
|
||
#: Incremental refresh by means of state tracking. | ||
STATE = "state" | ||
|
||
#: Automatic connectivity and stream init test via :ref:`--test<Test connectivity>`. | ||
TEST = "test" | ||
|
||
#: Support for ``replication_method: LOG_BASED``. You can read more about this | ||
#: feature in `MeltanoHub <https://hub.meltano.com/singer/docs#log-based>`_. | ||
LOG_BASED = "log-based" | ||
|
||
#: Deprecated. Please use :attr:`~TapCapabilities.CATALOG` instead. | ||
PROPERTIES = "properties", "Please use CATALOG instead." | ||
|
||
|
||
class TargetCapabilities(CapabilitiesEnum): | ||
"""Target-specific capabilities.""" | ||
|
||
#: Allows a ``soft_delete=True`` config option. | ||
#: Requires a tap stream supporting :attr:`PluginCapabilities.ACTIVATE_VERSION` | ||
#: and/or :attr:`TapCapabilities.LOG_BASED`. | ||
SOFT_DELETE = "soft-delete" | ||
|
||
#: Allows a ``hard_delete=True`` config option. | ||
#: Requires a tap stream supporting :attr:`PluginCapabilities.ACTIVATE_VERSION` | ||
#: and/or :attr:`TapCapabilities.LOG_BASED`. | ||
HARD_DELETE = "hard-delete" | ||
|
||
#: Fail safe for unknown JSON Schema types. | ||
DATATYPE_FAILSAFE = "datatype-failsafe" | ||
|
||
#: Allow de-nesting complex properties. | ||
RECORD_FLATTENING = "record-flattening" | ||
|
||
#: Allow setting the target schema. | ||
TARGET_SCHEMA = "target-schema" | ||
|
||
#: Validate the schema of the incoming records. | ||
VALIDATE_RECORDS = "validate-records" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,182 @@ | ||
"""Default JSON Schema to support config for built-in capabilities.""" | ||
|
||
from __future__ import annotations | ||
|
||
from singer_sdk.typing import ( | ||
ArrayType, | ||
BooleanType, | ||
IntegerType, | ||
NumberType, | ||
ObjectType, | ||
OneOf, | ||
PropertiesList, | ||
Property, | ||
StringType, | ||
) | ||
|
||
from ._enum import TargetLoadMethods | ||
|
||
STREAM_MAPS_CONFIG = PropertiesList( | ||
Property( | ||
"stream_maps", | ||
ObjectType(), | ||
description=( | ||
"Config object for stream maps capability. " | ||
"For more information check out " | ||
"[Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html)." | ||
), | ||
), | ||
Property( | ||
"stream_map_config", | ||
ObjectType(), | ||
description="User-defined config values to be used within map expressions.", | ||
), | ||
Property( | ||
"faker_config", | ||
ObjectType( | ||
Property( | ||
"seed", | ||
OneOf(NumberType, StringType, BooleanType), | ||
description=( | ||
"Value to seed the Faker generator for deterministic output: " | ||
"https://faker.readthedocs.io/en/master/#seeding-the-generator" | ||
), | ||
), | ||
Property( | ||
"locale", | ||
OneOf(StringType, ArrayType(StringType)), | ||
description=( | ||
"One or more LCID locale strings to produce localized output for: " | ||
"https://faker.readthedocs.io/en/master/#localization" | ||
), | ||
), | ||
), | ||
description=( | ||
"Config for the [`Faker`](https://faker.readthedocs.io/en/master/) " | ||
"instance variable `fake` used within map expressions. Only applicable if " | ||
"the plugin specifies `faker` as an additional dependency (through the " | ||
"`singer-sdk` `faker` extra or directly)." | ||
), | ||
), | ||
).to_dict() | ||
|
||
FLATTENING_CONFIG = PropertiesList( | ||
Property( | ||
"flattening_enabled", | ||
BooleanType(), | ||
description=( | ||
"'True' to enable schema flattening and automatically expand nested " | ||
"properties." | ||
), | ||
), | ||
Property( | ||
"flattening_max_depth", | ||
IntegerType(), | ||
description="The max depth to flatten schemas.", | ||
), | ||
).to_dict() | ||
|
||
BATCH_CONFIG = PropertiesList( | ||
Property( | ||
"batch_config", | ||
description="", | ||
wrapped=ObjectType( | ||
Property( | ||
"encoding", | ||
description="Specifies the format and compression of the batch files.", | ||
wrapped=ObjectType( | ||
Property( | ||
"format", | ||
StringType, | ||
allowed_values=["jsonl", "parquet"], | ||
description="Format to use for batch files.", | ||
), | ||
Property( | ||
"compression", | ||
StringType, | ||
allowed_values=["gzip", "none"], | ||
description="Compression format to use for batch files.", | ||
), | ||
), | ||
), | ||
Property( | ||
"storage", | ||
description="Defines the storage layer to use when writing batch files", | ||
wrapped=ObjectType( | ||
Property( | ||
"root", | ||
StringType, | ||
description="Root path to use when writing batch files.", | ||
), | ||
Property( | ||
"prefix", | ||
StringType, | ||
description="Prefix to use when writing batch files.", | ||
), | ||
), | ||
), | ||
), | ||
), | ||
).to_dict() | ||
|
||
TARGET_SCHEMA_CONFIG = PropertiesList( | ||
Property( | ||
"default_target_schema", | ||
StringType(), | ||
description="The default target database schema name to use for all streams.", | ||
), | ||
).to_dict() | ||
|
||
ADD_RECORD_METADATA_CONFIG = PropertiesList( | ||
Property( | ||
"add_record_metadata", | ||
BooleanType(), | ||
description="Add metadata to records.", | ||
), | ||
).to_dict() | ||
|
||
TARGET_HARD_DELETE_CONFIG = PropertiesList( | ||
Property( | ||
"hard_delete", | ||
BooleanType(), | ||
description="Hard delete records.", | ||
default=False, | ||
), | ||
).to_dict() | ||
|
||
TARGET_VALIDATE_RECORDS_CONFIG = PropertiesList( | ||
Property( | ||
"validate_records", | ||
BooleanType(), | ||
description="Whether to validate the schema of the incoming streams.", | ||
default=True, | ||
), | ||
).to_dict() | ||
|
||
TARGET_BATCH_SIZE_ROWS_CONFIG = PropertiesList( | ||
Property( | ||
"batch_size_rows", | ||
IntegerType, | ||
description="Maximum number of rows in each batch.", | ||
), | ||
).to_dict() | ||
|
||
TARGET_LOAD_METHOD_CONFIG = PropertiesList( | ||
Property( | ||
"load_method", | ||
StringType(), | ||
description=( | ||
"The method to use when loading data into the destination. " | ||
"`append-only` will always write all input records whether that records " | ||
"already exists or not. `upsert` will update existing records and insert " | ||
"new records. `overwrite` will delete all existing records and insert all " | ||
"input records." | ||
), | ||
allowed_values=[ | ||
TargetLoadMethods.APPEND_ONLY, | ||
TargetLoadMethods.UPSERT, | ||
TargetLoadMethods.OVERWRITE, | ||
], | ||
default=TargetLoadMethods.APPEND_ONLY, | ||
), | ||
).to_dict() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
"""Test the BuiltinSetting descriptor.""" | ||
|
||
from __future__ import annotations | ||
|
||
from singer_sdk.helpers.capabilities import ConfigProperty | ||
|
||
|
||
def test_builtin_setting_descriptor(): | ||
class ObjWithConfig: | ||
example = ConfigProperty(default=1) | ||
|
||
def __init__(self): | ||
self.config = {"example": 1} | ||
|
||
obj = ObjWithConfig() | ||
assert obj.example == 1 | ||
|
||
obj.config["example"] = 2 | ||
assert obj.example == 2 | ||
|
||
|
||
def test_builtin_setting_descriptor_custom_key(): | ||
class ObjWithConfig: | ||
my_attr = ConfigProperty("example", default=1) | ||
|
||
def __init__(self): | ||
self.config = {"example": 1} | ||
|
||
obj = ObjWithConfig() | ||
assert obj.my_attr == 1 | ||
|
||
obj.config["example"] = 2 | ||
assert obj.my_attr == 2 |