From 3bed39ff4cf7a228223975ed4327ece77f8c7947 Mon Sep 17 00:00:00 2001 From: Victor Mattos <5757883+vicmattos@users.noreply.github.com> Date: Mon, 20 Nov 2023 22:34:54 -0300 Subject: [PATCH 1/5] refactor(templates): simplify file names with `post_gen_project.py` hook --- cookiecutter/tap-template/cookiecutter.json | 4 +-- .../tap-template/hooks/post_gen_project.py | 35 +++++++++++++++++++ ...dependabot.yml{%endif%} => dependabot.yml} | 0 ... 'GitHub' %}test.yml{%endif%} => test.yml} | 0 ...ter.license %}LICENSE{%endif%} => LICENSE} | 0 .../{{{ 'test' }}_core.py => test_core.py} | 0 ...th2', 'JWT')%}auth.py{%endif%} => auth.py} | 0 ...%}client.py{%endif%} => graphql-client.py} | 5 ++- ...e %}client.py{%endif%} => other-client.py} | 0 ...pe %}client.py{%endif%} => rest-client.py} | 27 +++----------- ...ype %}client.py{%endif%} => sql-client.py} | 0 ..._type %}streams.py{%endif%} => streams.py} | 0 12 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 cookiecutter/tap-template/hooks/post_gen_project.py rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/{{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} => dependabot.yml} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} => test.yml} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} => LICENSE} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/{{{ 'test' }}_core.py => test_core.py} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} => auth.py} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} => graphql-client.py} (94%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} => other-client.py} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} => rest-client.py} (86%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} => sql-client.py} (100%) rename cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} => streams.py} (100%) diff --git a/cookiecutter/tap-template/cookiecutter.json b/cookiecutter/tap-template/cookiecutter.json index e297aae54..017b31109 100644 --- a/cookiecutter/tap-template/cookiecutter.json +++ b/cookiecutter/tap-template/cookiecutter.json @@ -14,8 +14,8 @@ "JWT", "Custom or N/A" ], - "include_ci_files": ["GitHub", "None (Skip)"], - "license": ["Apache-2.0"], + "include_ci_files": ["GitHub", "None"], + "license": ["Apache-2.0", "None"], "__prompts__": { "source_name": "The name of the source, in CamelCase", "admin_name": "Provide your [bold yellow]full name[/]", diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py new file mode 100644 index 000000000..900f509e9 --- /dev/null +++ b/cookiecutter/tap-template/hooks/post_gen_project.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +from pathlib import Path + + +BASE_PATH = Path('{{cookiecutter.library_name}}') + + +def delete_folder(pth: Path): + for sub in pth.iterdir(): + if sub.is_dir(): + delete_folder(sub) + else: + sub.unlink() + pth.rmdir() + + +if __name__ == '__main__': + + # Rename stream type client and delete others + target = Path(BASE_PATH, 'client.py') + Path(BASE_PATH, '{{cookiecutter.stream_type|lower}}-client.py').rename(target) + [c.unlink() for c in Path(BASE_PATH).rglob("*-client.py")] + + if '{{ cookiecutter.auth_method }}' not in ('OAuth2', 'JWT'): + Path(BASE_PATH, 'auth.py').unlink() + + if '{{ cookiecutter.stream_type }}' == 'SQL': + Path(BASE_PATH, 'streams.py').unlink() + + if '{{ cookiecutter.license }}' != 'Apache-2.0': + Path('LICENSE').unlink() + + if '{{ cookiecutter.include_ci_files }}' != 'GitHub': + delete_folder(Path('.github')) + diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/{% if cookiecutter.include_ci_files == 'GitHub' %}dependabot.yml{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/LICENSE similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{%if 'Apache-2.0' == cookiecutter.license %}LICENSE{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/LICENSE diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/{{ 'test' }}_core.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/test_core.py similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/{{ 'test' }}_core.py rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/test_core.py diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/auth.py similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/auth.py diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py similarity index 94% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py index 66505556d..f15664b22 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py @@ -5,15 +5,14 @@ from typing import Iterable import requests # noqa: TCH002 -from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream +from singer_sdk.streams import GraphQLStream {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} - from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator {%- endif %} -class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream): +class {{ cookiecutter.source_name }}Stream(GraphQLStream): """{{ cookiecutter.source_name }} stream class.""" @property diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/other-client.py similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/other-client.py diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py similarity index 86% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py index dae2269df..5bebd2393 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py @@ -3,43 +3,24 @@ from __future__ import annotations {% if cookiecutter.auth_method in ("OAuth2", "JWT") -%} -import sys -{% endif -%} +import sys{% endif -%} from pathlib import Path from typing import Any, Callable, Iterable import requests {% if cookiecutter.auth_method == "API Key" -%} from singer_sdk.authenticators import APIKeyAuthenticator -from singer_sdk.helpers.jsonpath import extract_jsonpath -from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 -from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream - {% elif cookiecutter.auth_method == "Bearer Token" -%} from singer_sdk.authenticators import BearerTokenAuthenticator -from singer_sdk.helpers.jsonpath import extract_jsonpath -from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 -from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream - {% elif cookiecutter.auth_method == "Basic Auth" -%} from singer_sdk.authenticators import BasicAuthenticator +{% endif -%} from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 -from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream - -{% elif cookiecutter.auth_method == "Custom or N/A" -%} -from singer_sdk.helpers.jsonpath import extract_jsonpath -from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 -from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream - -{% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} -from singer_sdk.helpers.jsonpath import extract_jsonpath -from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 -from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream +from singer_sdk.streams import RESTStream from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator -{% endif -%} {%- if cookiecutter.auth_method in ("OAuth2", "JWT") -%} if sys.version_info >= (3, 8): @@ -53,7 +34,7 @@ SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") -class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream): +class {{ cookiecutter.source_name }}Stream(RESTStream): """{{ cookiecutter.source_name }} stream class.""" @property diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py similarity index 100% rename from cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} rename to cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py From d4f910b168b71b0cecbfd84473ed0412f2cae260 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 01:36:00 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cookiecutter/tap-template/hooks/post_gen_project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py index 900f509e9..62cd5425c 100644 --- a/cookiecutter/tap-template/hooks/post_gen_project.py +++ b/cookiecutter/tap-template/hooks/post_gen_project.py @@ -29,7 +29,7 @@ def delete_folder(pth: Path): if '{{ cookiecutter.license }}' != 'Apache-2.0': Path('LICENSE').unlink() - + if '{{ cookiecutter.include_ci_files }}' != 'GitHub': delete_folder(Path('.github')) - + From 6f741449f30f9a1614bb6c7bc24a7b75bf70fa35 Mon Sep 17 00:00:00 2001 From: Victor Mattos <5757883+vicmattos@users.noreply.github.com> Date: Mon, 20 Nov 2023 23:49:10 -0300 Subject: [PATCH 3/5] revert(templates): undo changes within client files and only keep rename --- .../graphql-client.py | 5 ++-- .../rest-client.py | 27 ++++++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py index f15664b22..66505556d 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py @@ -5,14 +5,15 @@ from typing import Iterable import requests # noqa: TCH002 -from singer_sdk.streams import GraphQLStream +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} + from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator {%- endif %} -class {{ cookiecutter.source_name }}Stream(GraphQLStream): +class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream): """{{ cookiecutter.source_name }} stream class.""" @property diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py index 5bebd2393..dae2269df 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py @@ -3,24 +3,43 @@ from __future__ import annotations {% if cookiecutter.auth_method in ("OAuth2", "JWT") -%} -import sys{% endif -%} +import sys +{% endif -%} from pathlib import Path from typing import Any, Callable, Iterable import requests {% if cookiecutter.auth_method == "API Key" -%} from singer_sdk.authenticators import APIKeyAuthenticator +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + {% elif cookiecutter.auth_method == "Bearer Token" -%} from singer_sdk.authenticators import BearerTokenAuthenticator +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + {% elif cookiecutter.auth_method == "Basic Auth" -%} from singer_sdk.authenticators import BasicAuthenticator -{% endif -%} from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 -from singer_sdk.streams import RESTStream +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + +{% elif cookiecutter.auth_method == "Custom or N/A" -%} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + +{% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator +{% endif -%} {%- if cookiecutter.auth_method in ("OAuth2", "JWT") -%} if sys.version_info >= (3, 8): @@ -34,7 +53,7 @@ SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") -class {{ cookiecutter.source_name }}Stream(RESTStream): +class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream): """{{ cookiecutter.source_name }} stream class.""" @property From 9632460b9e2c2960ca74eb00d79e1148b8835ccf Mon Sep 17 00:00:00 2001 From: Victor Mattos <5757883+vicmattos@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:39:52 -0300 Subject: [PATCH 4/5] chore: remove template ci files from `check-yaml` pre-commit hook --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0aba49450..2f554df1b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,9 @@ repos: exclude: | (?x)^( cookiecutter/.*/meltano.yml| - cookiecutter/.*/.pre-commit-config.yaml + cookiecutter/.*/.pre-commit-config.yaml| + cookiecutter/.*/dependabot.yml| + cookiecutter/.*/test.yml )$ - id: end-of-file-fixer exclude: | From 5e2ab6e07a62952cb188bb25212817880ce9e3ff Mon Sep 17 00:00:00 2001 From: Victor Mattos <5757883+vicmattos@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:44:47 -0300 Subject: [PATCH 5/5] refactor(template): simplify folder remove in `post_gen_project.py` by using `shutil.rmtree` --- cookiecutter/tap-template/hooks/post_gen_project.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py index 62cd5425c..dee46e615 100644 --- a/cookiecutter/tap-template/hooks/post_gen_project.py +++ b/cookiecutter/tap-template/hooks/post_gen_project.py @@ -1,19 +1,11 @@ #!/usr/bin/env python from pathlib import Path +import shutil BASE_PATH = Path('{{cookiecutter.library_name}}') -def delete_folder(pth: Path): - for sub in pth.iterdir(): - if sub.is_dir(): - delete_folder(sub) - else: - sub.unlink() - pth.rmdir() - - if __name__ == '__main__': # Rename stream type client and delete others @@ -31,5 +23,5 @@ def delete_folder(pth: Path): Path('LICENSE').unlink() if '{{ cookiecutter.include_ci_files }}' != 'GitHub': - delete_folder(Path('.github')) + shutil.rmtree(Path('.github'))