diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..243590c --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,37 @@ +# Based on https://github.com/denkiwakame/py-tiny-pkg/blob/main/.github/workflows/pub.yml +# To create a release, see https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository + +name: publish + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v3 + with: + python-version: "3.x" + architecture: "x64" + - name: install pypa/build + run: >- + python -m + pip install + build + --user + - name: build sdist(tarball) and bdist(wheel) to dist/ + run: >- # = python -m build . works the same way by default + python -m + build + --sdist + --wheel + --outdir dist/ + - name: publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..2fdd7dc --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,55 @@ +# Based on https://github.com/denkiwakame/py-tiny-pkg/blob/main/.github/workflows/test.yml + +name: package + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + install-test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.8", "3.9"] + max-parallel: 3 + name: Python ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + architecture: "x64" + - name: confirm pip version + run: pip --version + - name: installation + run: pip install .[dev] + - name: test + run: python -m pytest --cov + editable-install-test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.8"] + max-parallel: 3 + name: Python ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + architecture: "x64" + - name: confirm pip version + run: pip --version + - name: installation + run: pip install -e .[dev] + - name: test + run: python -m pytest --cov diff --git a/.github/workflows/test_publish.yaml b/.github/workflows/test_publish.yaml new file mode 100644 index 0000000..c788e75 --- /dev/null +++ b/.github/workflows/test_publish.yaml @@ -0,0 +1,38 @@ +# Based on https://github.com/denkiwakame/py-tiny-pkg/blob/main/.github/workflows/testpub.yml + +name: publish-test + +on: + push: + tags: + - "*" + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v3 + with: + python-version: "3.x" + architecture: "x64" + - name: install pypa/build + run: >- + python -m + pip install + build + --user + - name: build sdist(tarball) and bdist(wheel) to dist/ + run: >- # = python -m build . works the same way by default + python -m + build + --sdist + --wheel + --outdir dist/ + - name: publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b1f752e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +Contributions to this repository are managed via the same process as for the [dss repository](https://github.com/interuss/dss/CONTRIBUTING.md). diff --git a/README.md b/README.md index 1fc4eb5..8caa33d 100644 --- a/README.md +++ b/README.md @@ -1 +1,15 @@ # implicitdict + +This library primarily provides the `ImplicitDict` base class which enables the inheriting class to implicitly be treated like a `dict` with entries corresponding to the fields of the inheriting class. Simple example: + +```python +class MyData(ImplicitDict): + foo: str + bar: int = 0 + baz: Optional[float] + +data: MyData = ImplicitDict.parse({'foo': 'asdf', 'bar': 1}, MyData) +assert json.dumps(data) == '{"foo": "asdf", "bar": 1}' +``` + +See [class documentation for `ImplicitDict`](src/implicitdict/__init__.py) and [test_normal_usage.py](tests/test_normal_usage.py) for more information. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a97e52f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,47 @@ +# Based on https://github.com/denkiwakame/py-tiny-pkg/blob/main/pyproject.toml + +[build-system] +requires = [ + "setuptools>=64", + "wheel", # for bdist package distribution + "setuptools_scm>=6.4", # for automated versioning +] + +build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = true + +[tool.setuptools_scm] +write_to = "src/implicitdict/_version.py" + +[project] +name = "implicitdict" +dynamic = ["version"] +authors = [ + { name="InterUSS Platform", email="tsc@lists.interussplatform.org" }, +] +description = "ImplicitDict base class that turns a subclass into a dict indexing attributes, making [de]serialization easy for complex typing-annotated data types." +readme = "README.md" +license = { file = "LICENSE.md" } +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = ["arrow", "pytimeparse"] +[project.optional-dependencies] +dev = ["pytest==5.0.0", "pytest-cov[all]", "black==21.10b0"] +[project.urls] +"Homepage" = "https://github.com/interuss/implicitdict" +"Bug Tracker" = "https://github.com/interuss/implicitdict/issues" + +[tool.black] +target-version = ['py39'] +line-length = 120 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a37d915 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +arrow==1.2.3 +pytimeparse==1.1.8 diff --git a/src/implicitdict/__init__.py b/src/implicitdict/__init__.py new file mode 100644 index 0000000..e55d1f0 --- /dev/null +++ b/src/implicitdict/__init__.py @@ -0,0 +1,238 @@ +import arrow +import datetime +from typing import get_args, get_origin, get_type_hints, Dict, Literal, Optional, Type, Union + +import pytimeparse + + +_DICT_FIELDS = set(dir({})) +_KEY_ALL_FIELDS = '_all_fields' +_KEY_OPTIONAL_FIELDS = '_optional_fields' + + +class ImplicitDict(dict): + """Base class that turns a subclass into a dict indexing attributes. + + The expected usage of this class is to declare a subclass with typed + attributes: + + class MySubclass1(ImplicitDict): + a: float + b: int = 2 + c: Optional[str] + d = 4 + + All non-optional attributes must be specified when constructing an instance of + the subclass. Untyped attributes with a default value are considered + optional. In the above example, an instance of MySubclass can only be + constructed when the values for `a` and `b` are specified in the constructor: + + MySubclass1() + >> ValueError + + MySubclass1(a=1) + >> ValueError + + MySubclass1(a=1, b=2) + + But, more values can be specified: + + MySubclass1(a=1, b=2, c='3', d='foo') + + The subclass allows access to all the declared attributes, but stores and + retrieves the attributes' values in and from the underlying dict. This means + that the subclass will always present itself as a dict with appropriate + entries for all declared attributes that are present. As a result, subclasses + are JSON-serializable by the default encoder. Example: + + x = MySubclass1(a=1, b=2) + x.d = 'foo' + import json + json.dumps(x) + >> '{"b": 2, "d": "foo", "a": 1}' + + To deserialize JSON into an ImplicitDict subclass, use ImplicitDict.parse: + + y: MySubclass1 = ImplicitDict.parse({'b': 2, 'd': 'foo', 'a': 1}, MySubclass1) + print(y.d) + >> foo + + If __init__ is overridden, ImplicitDict.__init__ should be called with + **kwargs. For example: + + class MySubclass2(ImplicitDict): + a: float + b: int = 2 + c: Optional[str] + d = 4 + + def __init__(self, **kwargs): + self.e = 5 + super(ImplicitDict, self).__init__(**kwargs) + """ + + @classmethod + def parse(cls, source: Dict, parse_type: Type): + if not isinstance(source, dict): + raise ValueError('Expected to find dictionary data to populate {} object but instead found {} type'.format(parse_type.__name__, type(source).__name__)) + kwargs = {} + hints = get_type_hints(parse_type) + for key, value in source.items(): + if key in hints: + # This entry has an explicit type + kwargs[key] = _parse_value(value, hints[key]) + else: + # This entry's type isn't specified + kwargs[key] = value + return parse_type(**kwargs) + + def __init__(self, previous_instance: Optional[dict]=None, **kwargs): + super(ImplicitDict, self).__init__() + subtype = type(self) + + if not hasattr(subtype, _KEY_ALL_FIELDS): + # Enumerate all fields and default values defined for the subclass + all_fields = set() + annotations = type(self).__annotations__ if hasattr(type(self), '__annotations__') else {} + for key in annotations: + all_fields.add(key) + + attributes = set() + for key in dir(self): + if key not in _DICT_FIELDS and key[0:2] != '__' and not callable(getattr(self, key)): + all_fields.add(key) + attributes.add(key) + + # Identify which fields are Optional + optional_fields = set() + for key, field_type in annotations.items(): + generic_type = get_origin(field_type) + if generic_type is Optional: + optional_fields.add(key) + elif generic_type is Union: + generic_args = get_args(field_type) + if len(generic_args) == 2 and generic_args[1] is type(None): + optional_fields.add(key) + for key in attributes: + if key not in annotations: + optional_fields.add(key) + + setattr(subtype, _KEY_ALL_FIELDS, all_fields) + setattr(subtype, _KEY_OPTIONAL_FIELDS, optional_fields) + else: + all_fields = getattr(subtype, _KEY_ALL_FIELDS) + optional_fields = getattr(subtype, _KEY_OPTIONAL_FIELDS) + + # Copy explicit field values passed to the constructor + provided_values = set() + if previous_instance: + for key, value in previous_instance.items(): + if key in all_fields: + self[key] = value + provided_values.add(key) + for key, value in kwargs.items(): + if key in all_fields: + if value is None and key in optional_fields and key not in provided_values: + # Don't consider an explicit null provided for an optional field as + # actually providing a value; instead, consider it omitting a value. + pass + else: + self[key] = value + provided_values.add(key) + + # Copy default field values + for key in all_fields: + if key not in provided_values: + if hasattr(type(self), key): + self[key] = super(ImplicitDict, self).__getattribute__(key) + + # Make sure all fields without a default and not labeled Optional were provided + for key in all_fields: + if key not in self and key not in optional_fields: + raise ValueError('Required field "{}" not specified in {}'.format(key, type(self).__name__)) + + def __getattribute__(self, item): + if hasattr(type(self), _KEY_ALL_FIELDS) and item in getattr(type(self), _KEY_ALL_FIELDS): + return self[item] + return super(ImplicitDict, self).__getattribute__(item) + + def __setattr__(self, key, value): + if hasattr(type(self), _KEY_ALL_FIELDS): + if key in getattr(type(self), _KEY_ALL_FIELDS): + self[key] = value + else: + raise KeyError('Attribute "{}" is not defined for "{}" object'.format(key, type(self).__name__)) + else: + super(ImplicitDict, self).__setattr__(key, value) + + def has_field_with_value(self, field_name: str) -> bool: + return field_name in self and self[field_name] is not None + + +def _parse_value(value, value_type: Type): + generic_type = get_origin(value_type) + if generic_type: + # Type is generic + arg_types = get_args(value_type) + if generic_type is list: + if issubclass(arg_types[0], ImplicitDict): + # value is a list of some kind of ImplicitDict values + return [ImplicitDict.parse(item, arg_types[0]) for item in value] + else: + # value is a list of non-ImplicitDict values + return value + + elif generic_type is dict: + # value is a dict of some kind + return {k if arg_types[0] is str else _parse_value(k, arg_types[0]): _parse_value(v, arg_types[1]) + for k, v in value.items()} + + elif generic_type is Union and len(arg_types) == 2 and arg_types[1] is type(None): + # Type is an Optional declaration + if value is None: + # An optional field specified explicitly as None is equivalent to + # omitting the field's value + return None + else: + return _parse_value(value, arg_types[0]) + + elif generic_type is Literal and len(arg_types) == 1: + # Type is a Literal (parsed value must match specified value) + if value != arg_types[0]: + raise ValueError('Value {} does not match required Literal {}'.format(value, arg_types[0])) + return value + + else: + raise NotImplementedError('Automatic parsing of {} type is not yet implemented'.format(value_type)) + + elif issubclass(value_type, ImplicitDict): + # value is an ImplicitDict + return ImplicitDict.parse(value, value_type) + + else: + # value is a non-generic type that is not an ImplicitDict + return value_type(value) if value_type else value + + +class StringBasedTimeDelta(str): + """String that only allows values which describe a timedelta.""" + def __new__(cls, value): + if isinstance(value, str): + dt = datetime.timedelta(seconds=pytimeparse.parse(value)) + else: + dt = value + str_value = str.__new__(cls, str(dt)) + str_value.timedelta = dt + return str_value + + +class StringBasedDateTime(str): + """String that only allows values which describe a datetime.""" + def __new__(cls, value): + if isinstance(value, str): + t = arrow.get(value).datetime + else: + t = value + str_value = str.__new__(cls, arrow.get(t).to('UTC').format('YYYY-MM-DDTHH:mm:ss.SSSSSS') + 'Z') + str_value.datetime = t + return str_value diff --git a/tests/test_normal_usage.py b/tests/test_normal_usage.py new file mode 100644 index 0000000..dd9d235 --- /dev/null +++ b/tests/test_normal_usage.py @@ -0,0 +1,126 @@ +from enum import Enum +import json +import pytest +from typing import Dict, List, Literal, Optional + +from implicitdict import ImplicitDict, StringBasedDateTime, StringBasedTimeDelta + + +class MyData(ImplicitDict): + foo: str + bar: int = 0 + baz: Optional[float] + + +def test_basic_usage(): + # Most basic usage is to parse a plain dict into an ImplicitDict... + data: MyData = ImplicitDict.parse({'foo': 'asdf', 'bar': 1}, MyData) + # ...and implicitly serialize the ImplicitDict to a plain dict + assert json.dumps(data) == '{"foo": "asdf", "bar": 1}' + + # Fields can be referenced directly... + assert data.foo == 'asdf' + assert data.bar == 1 + # ...or implicitly by name (because the object is implicitly a dict) + assert data['foo'] == 'asdf' + assert data['bar'] == 1 + + # Optional fields that aren't specified simply don't exist + assert 'baz' not in data + with pytest.raises(KeyError): + assert data.baz == 0 + + # Optional fields can be omitted (fields with defaults are optional) + data = MyData(foo='asdf') + assert json.loads(json.dumps(data)) == {'foo': 'asdf', 'bar': 0} + + # Optional fields can be specified + data = ImplicitDict.parse({'foo': 'asdf', 'baz': 1.23}, MyData) + assert json.loads(json.dumps(data)) == {'foo': 'asdf', 'bar': 0, 'baz': 1.23} + assert 'baz' in data + assert data.baz == 1.23 + + # Failing to specify a required field ("foo") raises a ValueError + with pytest.raises(ValueError): + MyData(bar=1) + + +class MyIntEnum(int, Enum): + Value1 = 1 + Value2 = 2 + Value3 = 3 + + +class MyStrEnum(str, Enum): + Value1 = 'foo' + Value2 = 'bar' + Value3 = 'baz' + + +class Features(ImplicitDict): + int_enum: MyIntEnum + str_enum: MyStrEnum + t_start: StringBasedDateTime + my_duration: StringBasedTimeDelta + my_literal: Literal['Must be this string'] + nested: Optional[MyData] + + +def test_features(): + src_dict = { + 'int_enum': 2, + 'str_enum': 'baz', + 't_start': '2022-01-01T01:23:45.6789Z', + 'my_duration': '1:23:45.67', + 'my_literal': 'Must be this string', + 'nested': { + 'foo': 'asdf' + }, + 'unrecognized_fields': 'are simply ignored' + } + data: Features = ImplicitDict.parse(src_dict, Features) + + assert data.int_enum == MyIntEnum.Value2 + assert data.int_enum == 2 + + assert data.str_enum == MyStrEnum.Value3 + assert data.str_enum == 'baz' + + assert data.t_start.datetime.year == 2022 + assert data.t_start.datetime.month == 1 + assert data.t_start.datetime.day == 1 + assert data.t_start.datetime.hour == 1 + assert data.t_start.datetime.minute == 23 + assert data.t_start.datetime.second == 45 + assert data.t_start.datetime.microsecond == 678900 + + assert data.my_duration.timedelta.total_seconds() == 1 * 3600 + 23 * 60 + 45.67 + + assert data.my_literal == 'Must be this string' + + assert 'nested' in data + + src_dict['my_literal'] = 'Not that string' + with pytest.raises(ValueError): + ImplicitDict.parse(src_dict, Features) + + +class NestedStructures(ImplicitDict): + my_list: List[MyData] + my_dict: Dict[str, List[float]] + + +def test_nested_structures(): + src_dict = { + 'my_list': [{'foo': 'one'}, {'foo': 'two'}], + 'my_dict': {'foo': 1.23, 'bar': 4.56} + } + data: NestedStructures = ImplicitDict.parse(src_dict, NestedStructures) + + assert len(data.my_list) == 2 + assert data.my_list[0].foo == 'one' + assert data.my_list[1].foo == 'two' + + assert len(data.my_dict) == 2 + assert data.my_dict['foo'] == 1.23 + assert data.my_dict['bar'] == 4.56