0.0.5a1 release

CHANGELOG.md

@@ -1,4 +1,14 @@
-[Go to GitHub Releases](https://github.com/dlt-hub/dlt-openapi/releases)
+[Go to GitHub Releases](https://github.com/dlt-hub/dlt-init-openapi/releases)
+
+0.0.4
+* Project rename to dlt-init-openapi
+* Pypi alpha release
+
+0.0.3
+* add param defaults & render not required query params
+* add support for paths with vars and file-ending
+* correct behavior for paginators with unknown total_path
+* auth improvements: select correct global auth, add secrets to secrets.toml, add note in readme if secret present
 
 0.0.2
 * Add warnings to logger for certain scenarios

Makefile

@@ -2,26 +2,28 @@
 
 
 update-rest-api:
-	poetry run python dlt_openapi/utils/update_rest_api.py
+	poetry run python dlt_init_openapi/utils/update_rest_api.py
 
+dev:
+	poetry install
 
 # lint
 lint: update-rest-api
 	rm -rf tests/_local
-	poetry run flake8 dlt_openapi tests
-	poetry run mypy dlt_openapi tests
-	poetry run black tests dlt_openapi --check
-	poetry run isort black tests dlt_openapi --check --diff
+	poetry run flake8 dlt_init_openapi tests
+	poetry run mypy dlt_init_openapi tests
+	poetry run black tests dlt_init_openapi --check
+	poetry run isort black tests dlt_init_openapi --check --diff
 
 # format the whole project
 format: update-rest-api
 	rm -rf tests/_local
-	poetry run isort black tests dlt_openapi
-	poetry run black tests dlt_openapi
+	poetry run isort black tests dlt_init_openapi
+	poetry run black tests dlt_init_openapi
 
 # all tests excluding the checks on e2e tests
 test: update-rest-api
-	poetry run python dlt_openapi/utils/update_rest_api.py
+	poetry run python dlt_init_openapi/utils/update_rest_api.py
 	poetry run pytest tests --ignore=tests/e2e
 
 # test without running all the specs through a source
@@ -30,17 +32,24 @@ test-fast: update-rest-api
 
 # dev helpers
 create-pokemon-pipeline:
-	poetry run dlt-openapi init pokemon --url https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml --no-interactive
+	poetry run dlt-init-openapi init pokemon --url https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml --no-interactive
 
 create-pokemon-pipeline-interactive:
-	poetry run dlt-openapi init pokemon --url https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml
+	poetry run dlt-init-openapi init pokemon --url https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml
 
 # e2e test helpers
 create-e2e-pokemon-pipeline:
-	poetry run dlt-openapi init pokemon --path tests/cases/e2e_specs/pokeapi.yml --global-limit 2 --no-interactive
+	poetry run dlt-init-openapi init pokemon --path tests/cases/e2e_specs/pokeapi.yml --global-limit 2 --no-interactive
 
 run-pokemon-pipeline:
 	cd pokemon-pipeline && poetry run python pipeline.py
 
 check-pokemon-pipeline:
-	poetry run pytest tests/e2e
+	poetry run pytest tests/e2e
+
+build-library: dev
+	poetry version
+	poetry build
+
+publish-library: build-library
+	poetry publish

README.md

@@ -1,5 +1,5 @@
-# dlt-openapi
-`dlt-openapi` generates [`dlt`](https://dlthub.com/docs) pipelines from OpenAPI 3.x documents/specs using the [`dlt` `rest_api` `verified source`](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api). If you do not know `dlt` or our `verified sources`, please read:
+# dlt-init-openapi
+`dlt-init-openapi` generates [`dlt`](https://dlthub.com/docs) pipelines from OpenAPI 3.x documents/specs using the [`dlt` `rest_api` `verified source`](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api). If you do not know `dlt` or our `verified sources`, please read:
 
 * [Getting started](https://dlthub.com/docs/getting-started) to learn the `dlt` basics
 * [dlt rest_api](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api) to learn how our `rest_api` source works
@@ -13,7 +13,7 @@ This project started as a fork of [openapi-python-client](https://github.com/ope
 
 
 ## Features
-The dlt-openapi generates code from an OpenAPI spec that you can use to extract data from a `rest_api` into any [`destination`](https://dlthub.com/docs/dlt-ecosystem/destinations/) (e.g. Postgres, BigQuery, Redshift...) `dlt` supports.
+The dlt-init-openapi generates code from an OpenAPI spec that you can use to extract data from a `rest_api` into any [`destination`](https://dlthub.com/docs/dlt-ecosystem/destinations/) (e.g. Postgres, BigQuery, Redshift...) `dlt` supports.
 
 Features include
 
@@ -29,7 +29,7 @@ You will need Python 3.9 installed, as well as [`poetry`](https://python-poetry.
 
 ```console
 # 1. Checkout this repository locally
-$ git clone [email protected]:dlt-hub/dlt-openapi.git
+$ git clone [email protected]:dlt-hub/dlt-init-openapi.git
 
 # 2. Install required poetry dependencies
 $ poetry install
@@ -43,8 +43,8 @@ $ poetry shell
 Let's create an example pipeline from the [PokeAPI spec](https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml). You can point to any other OpenAPI Spec instead if you like.
 
 ```console
-# 1. Run the generator with the dlt-openapi init command:
-$ dlt-openapi init pokemon --url https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml
+# 1. Run the generator with the dlt-init-openapi init command:
+$ dlt-init-openapi init pokemon --url https://raw.githubusercontent.com/cliffano/pokeapi-clients/ec9a2707ef2a85f41b747d8df013e272ef650ec5/specification/pokeapi.yml
 
 # 2. You can now pick the endpoints you need from the popup
 
@@ -90,14 +90,14 @@ $ dlt-init init pokemon --path ./path/to/my_spec.yml
 
 - `init`: Generate a new `dlt` `rest_api` `source`
 
-### `dlt-openapi init`
+### `dlt-init-openapi init`
 
 Generate a new `dlt` `rest_api` `source`
 
 **Usage**:
 
 ```console
-$ dlt-openapi init pokemon --path ./path/to/my_spec.yml
+$ dlt-init-openapi init pokemon --path ./path/to/my_spec.yml
 ```
 
 **Options**:
@@ -111,7 +111,7 @@ $ dlt-openapi init pokemon --path ./path/to/my_spec.yml
 - `--help`: Show this message and exit.
 
 ## Config options
-You can pass a path to a config file with the `--config PATH` argument. To see available config values, go to https://github.com/dlt-hub/dlt-openapi/blob/master/dlt_openapi/config.py and read the information below each field on the `Config` class.
+You can pass a path to a config file with the `--config PATH` argument. To see available config values, go to https://github.com/dlt-hub/dlt-init-openapi/blob/master/dlt_init_openapi/config.py and read the information below each field on the `Config` class.
 
 The config file can be supplied as json or yaml dictionary. For example to change the package name, you can create a yaml file:
 
@@ -123,7 +123,7 @@ package_name: "other_package_name"
 And use it with the config argument:
 
 ```console
-$ dlt-openapi init pokemon --url ... --config config.yml
+$ dlt-init-openapi init pokemon --url ... --config config.yml
 ```
 
 ## Implementation notes

dlt_openapi/__init__.py → dlt_init_openapi/__init__.py

@@ -9,7 +9,7 @@
 import httpx
 from loguru import logger
 
-from dlt_openapi.utils.misc import import_class_from_string
+from dlt_init_openapi.utils.misc import import_class_from_string
 
 from .config import Config
 from .detector.base_detector import GLOBAL_WARNING_KEY, BaseDetector

dlt_openapi/cli/__init__.py → dlt_init_openapi/cli/__init__.py

@@ -6,18 +6,18 @@
 import typer
 from loguru import logger
 
-from dlt_openapi.cli.cli_endpoint_selection import questionary_endpoint_selection
-from dlt_openapi.config import Config
-from dlt_openapi.utils import update_rest_api
+from dlt_init_openapi.cli.cli_endpoint_selection import questionary_endpoint_selection
+from dlt_init_openapi.config import Config
+from dlt_init_openapi.utils import update_rest_api
 
 app = typer.Typer()
 
 
 def _print_version(value: bool) -> None:
-    from dlt_openapi import __version__
+    from dlt_init_openapi import __version__
 
     if value:
-        typer.echo(f"dlt-openapi version: {__version__}")
+        typer.echo(f"dlt-init-openapi version: {__version__}")
         raise typer.Exit()
 
 
@@ -34,7 +34,7 @@ def _load_config(path: Optional[pathlib.Path], config: Any) -> Config:
 
 # noinspection PyUnusedLocal
 # pylint: disable=unused-argument
-@app.callback(name="dlt-openapi")
+@app.callback(name="dlt-init-openapi")
 def cli(
     version: bool = typer.Option(False, "--version", callback=_print_version, help="Print the version and exit"),
 ) -> None:
@@ -60,7 +60,7 @@ def init(
     ),
 ) -> None:
     """Generate a new OpenAPI Client library"""
-    from dlt_openapi import create_new_client
+    from dlt_init_openapi import create_new_client
 
     # set up console logging
     logger.remove()

dlt_openapi/cli/cli_endpoint_selection.py → dlt_init_openapi/cli/cli_endpoint_selection.py

@@ -2,7 +2,7 @@
 
 import questionary
 
-from dlt_openapi.parser.endpoints import Endpoint, EndpointCollection
+from dlt_init_openapi.parser.endpoints import Endpoint, EndpointCollection
 
 
 def questionary_endpoint_selection(endpoints: EndpointCollection) -> Set[str]:

dlt_openapi/config.py → dlt_init_openapi/config.py

@@ -39,9 +39,9 @@ class Config(BaseModel):
     """filter for endpoint rendering"""
     name_resources_by_operation: bool = False
     """always name resources by operation id, useful for testing"""
-    renderer_class: str = "dlt_openapi.renderer.default.DefaultRenderer"
+    renderer_class: str = "dlt_init_openapi.renderer.default.DefaultRenderer"
     """Which class to use for rendering"""
-    detector_class: str = "dlt_openapi.detector.default.DefaultDetector"
+    detector_class: str = "dlt_init_openapi.detector.default.DefaultDetector"
     """Which class to use for detecting"""
     global_limit: int = 0
     """Set a limit on how many items are emitted from a resource"""

dlt_openapi/detector/base_detector.py → dlt_init_openapi/detector/base_detector.py

@@ -6,7 +6,7 @@
 from typing import TYPE_CHECKING, Dict, List
 
 if TYPE_CHECKING:
-    from dlt_openapi.parser.openapi_parser import OpenapiParser
+    from dlt_init_openapi.parser.openapi_parser import OpenapiParser
 
 GLOBAL_WARNING_KEY = "global"
 

dlt_openapi/detector/default/__init__.py → dlt_init_openapi/detector/default/__init__.py

@@ -5,17 +5,17 @@
 import json
 from typing import Dict, List, Optional, Tuple, Union, cast
 
-from dlt_openapi.config import Config
-from dlt_openapi.detector.base_detector import GLOBAL_WARNING_KEY, BaseDetector
-from dlt_openapi.detector.default import utils
-from dlt_openapi.detector.default.primary_key import detect_primary_key_by_name
-from dlt_openapi.parser.endpoints import Endpoint, EndpointCollection, Response, TransformerSetting
-from dlt_openapi.parser.models import DataPropertyPath
-from dlt_openapi.parser.openapi_parser import OpenapiParser
-from dlt_openapi.parser.pagination import Pagination
-from dlt_openapi.parser.parameters import Parameter
-from dlt_openapi.utils.misc import snake_case
-from dlt_openapi.utils.paths import (
+from dlt_init_openapi.config import Config
+from dlt_init_openapi.detector.base_detector import GLOBAL_WARNING_KEY, BaseDetector
+from dlt_init_openapi.detector.default import utils
+from dlt_init_openapi.detector.default.primary_key import detect_primary_key_by_name
+from dlt_init_openapi.parser.endpoints import Endpoint, EndpointCollection, Response, TransformerSetting
+from dlt_init_openapi.parser.models import DataPropertyPath
+from dlt_init_openapi.parser.openapi_parser import OpenapiParser
+from dlt_init_openapi.parser.pagination import Pagination
+from dlt_init_openapi.parser.parameters import Parameter
+from dlt_init_openapi.utils.misc import snake_case
+from dlt_init_openapi.utils.paths import (
     get_non_var_path_parts,
     get_path_parts,
     get_path_var_names,

dlt_openapi/detector/default/primary_key.py → dlt_init_openapi/detector/default/primary_key.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from dlt_openapi.utils import paths
+from dlt_init_openapi.utils import paths
 
 from .const import PRIMARY_KEY_SUFFIXES, PRIMARY_KEY_WORD_SEPARATORS, PRIMAY_KEY_NAMES
 from .utils import get_word_variations

dlt_openapi/detector/default/utils.py → dlt_init_openapi/detector/default/utils.py

@@ -4,7 +4,7 @@
 
 import inflector  # type: ignore
 
-from dlt_openapi.utils import paths
+from dlt_init_openapi.utils import paths
 
 inf = inflector.English()
 

dlt_openapi/detector/default/warnings.py → dlt_init_openapi/detector/default/warnings.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from dlt_openapi.detector.base_detector import BaseDetectionWarning
+from dlt_init_openapi.detector.base_detector import BaseDetectionWarning
 
 
 class PrimaryKeyNotFoundWarning(BaseDetectionWarning):

dlt_init_openapi/parser/config.py

@@ -0,0 +1 @@
+from dlt_init_openapi.config import Config  # noqa: F401

dlt_openapi/parser/context.py → dlt_init_openapi/parser/context.py

@@ -4,7 +4,7 @@
 import referencing
 import referencing.jsonschema
 
-from dlt_openapi.parser.config import Config
+from dlt_init_openapi.parser.config import Config
 
 TComponentClass = Union[
     osp.Schema,

dlt_openapi/parser/endpoints.py → dlt_init_openapi/parser/endpoints.py

@@ -6,11 +6,11 @@
 import openapi_schema_pydantic as osp
 from loguru import logger
 
-from dlt_openapi.parser.context import OpenapiContext
-from dlt_openapi.parser.models import DataPropertyPath, SchemaWrapper
-from dlt_openapi.parser.pagination import Pagination
-from dlt_openapi.parser.parameters import Parameter
-from dlt_openapi.utils.paths import get_path_var_names, path_looks_like_list
+from dlt_init_openapi.parser.context import OpenapiContext
+from dlt_init_openapi.parser.models import DataPropertyPath, SchemaWrapper
+from dlt_init_openapi.parser.pagination import Pagination
+from dlt_init_openapi.parser.parameters import Parameter
+from dlt_init_openapi.utils.paths import get_path_var_names, path_looks_like_list
 
 TMethod = Literal["GET", "POST", "PUT", "PATCH"]
 

dlt_openapi/parser/info.py → dlt_init_openapi/parser/info.py

@@ -3,7 +3,7 @@
 
 import openapi_schema_pydantic as osp
 
-from dlt_openapi.parser.context import OpenapiContext
+from dlt_init_openapi.parser.context import OpenapiContext
 
 
 @dataclass

dlt_openapi/parser/models.py → dlt_init_openapi/parser/models.py

@@ -8,11 +8,11 @@
 import openapi_schema_pydantic as osp
 from dlt.common.utils import digest128
 
-from dlt_openapi.parser.types import DataType, TOpenApiType
-from dlt_openapi.utils.misc import unique_list
+from dlt_init_openapi.parser.types import DataType, TOpenApiType
+from dlt_init_openapi.utils.misc import unique_list
 
 if TYPE_CHECKING:
-    from dlt_openapi.parser.context import OpenapiContext
+    from dlt_init_openapi.parser.context import OpenapiContext
 
 from .const import MAX_RECURSION_DEPTH
 

dlt_openapi/parser/openapi_parser.py → dlt_init_openapi/parser/openapi_parser.py

@@ -7,13 +7,13 @@
 from loguru import logger
 from yaml import BaseLoader
 
-from dlt_openapi.exceptions import DltOpenAPIException
-from dlt_openapi.parser.config import Config
-from dlt_openapi.parser.context import OpenapiContext
-from dlt_openapi.parser.endpoints import EndpointCollection
-from dlt_openapi.parser.info import OpenApiInfo
-from dlt_openapi.parser.pagination import Pagination
-from dlt_openapi.parser.security import SecurityScheme
+from dlt_init_openapi.exceptions import DltOpenAPIException
+from dlt_init_openapi.parser.config import Config
+from dlt_init_openapi.parser.context import OpenapiContext
+from dlt_init_openapi.parser.endpoints import EndpointCollection
+from dlt_init_openapi.parser.info import OpenApiInfo
+from dlt_init_openapi.parser.pagination import Pagination
+from dlt_init_openapi.parser.security import SecurityScheme
 
 
 class OpenapiParser:

dlt_openapi/parser/pagination.py → dlt_init_openapi/parser/pagination.py

@@ -2,7 +2,7 @@
 from typing import TYPE_CHECKING, Dict, List, Union
 
 if TYPE_CHECKING:
-    from dlt_openapi.parser.endpoints import Parameter
+    from dlt_init_openapi.parser.endpoints import Parameter
 
 
 @dataclass

dlt_openapi/parser/parameters.py → dlt_init_openapi/parser/parameters.py

@@ -3,9 +3,9 @@
 
 import openapi_schema_pydantic as osp
 
-from dlt_openapi.parser.context import OpenapiContext
-from dlt_openapi.parser.models import DataPropertyPath, SchemaWrapper, TSchemaType
-from dlt_openapi.parser.types import compare_openapi_types
+from dlt_init_openapi.parser.context import OpenapiContext
+from dlt_init_openapi.parser.models import DataPropertyPath, SchemaWrapper, TSchemaType
+from dlt_init_openapi.parser.types import compare_openapi_types
 
 TParamIn = Literal["query", "header", "path", "cookie"]
 

dlt_openapi/parser/types.py → dlt_init_openapi/parser/types.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING, Dict, List, Literal, Optional
 
 if TYPE_CHECKING:
-    from dlt_openapi.parser.models import Property, SchemaWrapper
+    from dlt_init_openapi.parser.models import Property, SchemaWrapper
 
 
 TOpenApiType = Literal["boolean", "object", "array", "number", "string", "integer"]