From c49868301ab3f137b98d9eb48314dfc32d0c974a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Nov 2023 08:57:45 -0600 Subject: [PATCH 1/5] chore(deps): bump poetry from 1.7.0 to 1.7.1 in /.github/workflows (#2051) Bumps [poetry](https://github.com/python-poetry/poetry) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/python-poetry/poetry/releases) - [Changelog](https://github.com/python-poetry/poetry/blob/master/CHANGELOG.md) - [Commits](https://github.com/python-poetry/poetry/compare/1.7.0...1.7.1) --- updated-dependencies: - dependency-name: poetry dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index b0c49c1b0..8ee1d08be 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,5 +1,5 @@ pip==23.3.1 -poetry==1.7.0 +poetry==1.7.1 pre-commit==3.5.0 nox==2023.4.22 nox-poetry==1.0.3 From 80e31a26db4ddd88e4dbee1dcc9357e7e4f913b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Nov 2023 08:58:00 -0600 Subject: [PATCH 2/5] chore(deps-dev): bump duckdb from 0.9.1 to 0.9.2 (#2050) Bumps [duckdb](https://github.com/duckdb/duckdb) from 0.9.1 to 0.9.2. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Changelog](https://github.com/duckdb/duckdb/blob/main/tools/release-pip.py) - [Commits](https://github.com/duckdb/duckdb/compare/v0.9.1...v0.9.2) --- updated-dependencies: - dependency-name: duckdb dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 82 ++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/poetry.lock b/poetry.lock index 891d5e0a8..813936141 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -537,50 +537,50 @@ files = [ [[package]] name = "duckdb" -version = "0.9.1" +version = "0.9.2" description = "DuckDB embedded database" optional = false python-versions = ">=3.7.0" files = [ - {file = "duckdb-0.9.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6c724e105ecd78c8d86b3c03639b24e1df982392fc836705eb007e4b1b488864"}, - {file = "duckdb-0.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:75f12c5a3086079fb6440122565f1762ef1a610a954f2d8081014c1dd0646e1a"}, - {file = "duckdb-0.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:151f5410c32f8f8fe03bf23462b9604349bc0b4bd3a51049bbf5e6a482a435e8"}, - {file = "duckdb-0.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c1d066fdae22b9b711b1603541651a378017645f9fbc4adc9764b2f3c9e9e4a"}, - {file = "duckdb-0.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1de56d8b7bd7a7653428c1bd4b8948316df488626d27e9c388194f2e0d1428d4"}, - {file = "duckdb-0.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1fb6cd590b1bb4e31fde8efd25fedfbfa19a86fa72789fa5b31a71da0d95bce4"}, - {file = "duckdb-0.9.1-cp310-cp310-win32.whl", hash = "sha256:1039e073714d668cef9069bb02c2a6756c7969cedda0bff1332520c4462951c8"}, - {file = "duckdb-0.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:7e6ac4c28918e1d278a89ff26fd528882aa823868ed530df69d6c8a193ae4e41"}, - {file = "duckdb-0.9.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5eb750f2ee44397a61343f32ee9d9e8c8b5d053fa27ba4185d0e31507157f130"}, - {file = "duckdb-0.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aea2a46881d75dc069a242cb164642d7a4f792889010fb98210953ab7ff48849"}, - {file = "duckdb-0.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed3dcedfc7a9449b6d73f9a2715c730180056e0ba837123e7967be1cd3935081"}, - {file = "duckdb-0.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c55397bed0087ec4445b96f8d55f924680f6d40fbaa7f2e35468c54367214a5"}, - {file = "duckdb-0.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3261696130f1cfb955735647c93297b4a6241753fb0de26c05d96d50986c6347"}, - {file = "duckdb-0.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:64c04b1728e3e37cf93748829b5d1e028227deea75115bb5ead01c608ece44b1"}, - {file = "duckdb-0.9.1-cp311-cp311-win32.whl", hash = "sha256:12cf9fb441a32702e31534330a7b4d569083d46a91bf185e0c9415000a978789"}, - {file = "duckdb-0.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:fdfd85575ce9540e593d5d25c9d32050bd636c27786afd7b776aae0f6432b55e"}, - {file = "duckdb-0.9.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:704700a4b469e3bb1a7e85ac12e58037daaf2b555ef64a3fe2913ffef7bd585b"}, - {file = "duckdb-0.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf55b303b7b1a8c2165a96e609eb30484bc47481d94a5fb1e23123e728df0a74"}, - {file = "duckdb-0.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b70e23c14746904ca5de316436e43a685eb769c67fe3dbfaacbd3cce996c5045"}, - {file = "duckdb-0.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:77379f7f1f8b4dc98e01f8f6f8f15a0858cf456e2385e22507f3cb93348a88f9"}, - {file = "duckdb-0.9.1-cp37-cp37m-win32.whl", hash = "sha256:92c8f738489838666cae9ef41703f8b16f660bb146970d1eba8b2c06cb3afa39"}, - {file = "duckdb-0.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08c5484ac06ab714f745526d791141f547e2f5ac92f97a0a1b37dfbb3ea1bd13"}, - {file = "duckdb-0.9.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f66d3c07c7f6938d3277294677eb7dad75165e7c57c8dd505503fc5ef10f67ad"}, - {file = "duckdb-0.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c38044e5f78c0c7b58e9f937dcc6c34de17e9ca6be42f9f8f1a5a239f7a847a5"}, - {file = "duckdb-0.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:73bc0d715b79566b3ede00c367235cfcce67be0eddda06e17665c7a233d6854a"}, - {file = "duckdb-0.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d26622c3b4ea6a8328d95882059e3cc646cdc62d267d48d09e55988a3bba0165"}, - {file = "duckdb-0.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3367d10096ff2b7919cedddcf60d308d22d6e53e72ee2702f6e6ca03d361004a"}, - {file = "duckdb-0.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d88a119f1cb41911a22f08a6f084d061a8c864e28b9433435beb50a56b0d06bb"}, - {file = "duckdb-0.9.1-cp38-cp38-win32.whl", hash = "sha256:99567496e45b55c67427133dc916013e8eb20a811fc7079213f5f03b2a4f5fc0"}, - {file = "duckdb-0.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:5b3da4da73422a3235c3500b3fb541ac546adb3e35642ef1119dbcd9cc7f68b8"}, - {file = "duckdb-0.9.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eca00c0c2062c0265c6c0e78ca2f6a30611b28f3afef062036610e9fc9d4a67d"}, - {file = "duckdb-0.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eb5af8e89d40fc4baab1515787ea1520a6c6cf6aa40ab9f107df6c3a75686ce1"}, - {file = "duckdb-0.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fae3d4f83ebcb47995f6acad7c6d57d003a9b6f0e1b31f79a3edd6feb377443"}, - {file = "duckdb-0.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16b9a7efc745bc3c5d1018c3a2f58d9e6ce49c0446819a9600fdba5f78e54c47"}, - {file = "duckdb-0.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b0b60167f5537772e9f5af940e69dcf50e66f5247732b8bb84a493a9af6055"}, - {file = "duckdb-0.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4f27f5e94c47df6c4ccddf18e3277b7464eea3db07356d2c4bf033b5c88359b8"}, - {file = "duckdb-0.9.1-cp39-cp39-win32.whl", hash = "sha256:d43cd7e6f783006b59dcc5e40fcf157d21ee3d0c8dfced35278091209e9974d7"}, - {file = "duckdb-0.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:e666795887d9cf1d6b6f6cbb9d487270680e5ff6205ebc54b2308151f13b8cff"}, - {file = "duckdb-0.9.1.tar.gz", hash = "sha256:603a878746015a3f2363a65eb48bcbec816261b6ee8d71eee53061117f6eef9d"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:aadcea5160c586704c03a8a796c06a8afffbefefb1986601104a60cb0bfdb5ab"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:08215f17147ed83cbec972175d9882387366de2ed36c21cbe4add04b39a5bcb4"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee6c2a8aba6850abef5e1be9dbc04b8e72a5b2c2b67f77892317a21fae868fe7"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ff49f3da9399900fd58b5acd0bb8bfad22c5147584ad2427a78d937e11ec9d0"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5ac5baf8597efd2bfa75f984654afcabcd698342d59b0e265a0bc6f267b3f0"}, + {file = "duckdb-0.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:81c6df905589a1023a27e9712edb5b724566587ef280a0c66a7ec07c8083623b"}, + {file = "duckdb-0.9.2-cp310-cp310-win32.whl", hash = "sha256:a298cd1d821c81d0dec8a60878c4b38c1adea04a9675fb6306c8f9083bbf314d"}, + {file = "duckdb-0.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:492a69cd60b6cb4f671b51893884cdc5efc4c3b2eb76057a007d2a2295427173"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:061a9ea809811d6e3025c5de31bc40e0302cfb08c08feefa574a6491e882e7e8"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a43f93be768af39f604b7b9b48891f9177c9282a408051209101ff80f7450d8f"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ac29c8c8f56fff5a681f7bf61711ccb9325c5329e64f23cb7ff31781d7b50773"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b14d98d26bab139114f62ade81350a5342f60a168d94b27ed2c706838f949eda"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:796a995299878913e765b28cc2b14c8e44fae2f54ab41a9ee668c18449f5f833"}, + {file = "duckdb-0.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6cb64ccfb72c11ec9c41b3cb6181b6fd33deccceda530e94e1c362af5f810ba1"}, + {file = "duckdb-0.9.2-cp311-cp311-win32.whl", hash = "sha256:930740cb7b2cd9e79946e1d3a8f66e15dc5849d4eaeff75c8788d0983b9256a5"}, + {file = "duckdb-0.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:c28f13c45006fd525001b2011cdf91fa216530e9751779651e66edc0e446be50"}, + {file = "duckdb-0.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fbce7bbcb4ba7d99fcec84cec08db40bc0dd9342c6c11930ce708817741faeeb"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15a82109a9e69b1891f0999749f9e3265f550032470f51432f944a37cfdc908b"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9490fb9a35eb74af40db5569d90df8a04a6f09ed9a8c9caa024998c40e2506aa"}, + {file = "duckdb-0.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:696d5c6dee86c1a491ea15b74aafe34ad2b62dcd46ad7e03b1d00111ca1a8c68"}, + {file = "duckdb-0.9.2-cp37-cp37m-win32.whl", hash = "sha256:4f0935300bdf8b7631ddfc838f36a858c1323696d8c8a2cecbd416bddf6b0631"}, + {file = "duckdb-0.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:0aab900f7510e4d2613263865570203ddfa2631858c7eb8cbed091af6ceb597f"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d8130ed6a0c9421b135d0743705ea95b9a745852977717504e45722c112bf7a"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:974e5de0294f88a1a837378f1f83330395801e9246f4e88ed3bfc8ada65dcbee"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4fbc297b602ef17e579bb3190c94d19c5002422b55814421a0fc11299c0c1100"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dd58a0d84a424924a35b3772419f8cd78a01c626be3147e4934d7a035a8ad68"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11a1194a582c80dfb57565daa06141727e415ff5d17e022dc5f31888a5423d33"}, + {file = "duckdb-0.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:be45d08541002a9338e568dca67ab4f20c0277f8f58a73dfc1435c5b4297c996"}, + {file = "duckdb-0.9.2-cp38-cp38-win32.whl", hash = "sha256:dd6f88aeb7fc0bfecaca633629ff5c986ac966fe3b7dcec0b2c48632fd550ba2"}, + {file = "duckdb-0.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:28100c4a6a04e69aa0f4a6670a6d3d67a65f0337246a0c1a429f3f28f3c40b9a"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7ae5bf0b6ad4278e46e933e51473b86b4b932dbc54ff097610e5b482dd125552"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5d0bb845a80aa48ed1fd1d2d285dd352e96dc97f8efced2a7429437ccd1fe1f"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ce262d74a52500d10888110dfd6715989926ec936918c232dcbaddb78fc55b4"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6935240da090a7f7d2666f6d0a5e45ff85715244171ca4e6576060a7f4a1200e"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5cfb93e73911696a98b9479299d19cfbc21dd05bb7ab11a923a903f86b4d06e"}, + {file = "duckdb-0.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:64e3bc01751f31e7572d2716c3e8da8fe785f1cdc5be329100818d223002213f"}, + {file = "duckdb-0.9.2-cp39-cp39-win32.whl", hash = "sha256:6e5b80f46487636368e31b61461940e3999986359a78660a50dfdd17dd72017c"}, + {file = "duckdb-0.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:e6142a220180dbeea4f341708bd5f9501c5c962ce7ef47c1cadf5e8810b4cb13"}, + {file = "duckdb-0.9.2.tar.gz", hash = "sha256:3843afeab7c3fc4a4c0b53686a4cc1d9cdbdadcbb468d60fef910355ecafd447"}, ] [[package]] From b8ab97ebdd7a8d42d6e0578f591aebf28ed481ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:08:26 -0600 Subject: [PATCH 3/5] chore: Fix some docstring typos and apply refactors suggested by Sourcery (#2052) --- singer_sdk/_singerlib/catalog.py | 8 ++------ singer_sdk/authenticators.py | 22 +++++++++------------- singer_sdk/connectors/sql.py | 3 --- singer_sdk/helpers/_conformers.py | 12 +++++++----- singer_sdk/helpers/_flattening.py | 7 ++----- singer_sdk/helpers/_state.py | 15 ++++----------- singer_sdk/helpers/_typing.py | 7 ++++--- singer_sdk/io_base.py | 2 +- singer_sdk/mapper.py | 7 ++----- singer_sdk/metrics.py | 7 +++---- singer_sdk/sinks/core.py | 2 +- singer_sdk/sinks/sql.py | 8 +------- singer_sdk/tap_base.py | 4 ++-- singer_sdk/target_base.py | 2 +- singer_sdk/testing/factory.py | 1 + singer_sdk/testing/legacy.py | 5 +---- singer_sdk/typing.py | 23 +++++++++++------------ 17 files changed, 52 insertions(+), 83 deletions(-) diff --git a/singer_sdk/_singerlib/catalog.py b/singer_sdk/_singerlib/catalog.py index 77fe884d8..87b528466 100644 --- a/singer_sdk/_singerlib/catalog.py +++ b/singer_sdk/_singerlib/catalog.py @@ -31,11 +31,7 @@ def __missing__(self, breadcrumb: Breadcrumb) -> bool: Returns: True if the breadcrumb is selected, False otherwise. """ - if len(breadcrumb) >= 2: # noqa: PLR2004 - parent = breadcrumb[:-2] - return self[parent] - - return True + return self[breadcrumb[:-2]] if len(breadcrumb) >= 2 else True # noqa: PLR2004 @dataclass @@ -71,7 +67,7 @@ def from_dict(cls: type[Metadata], value: dict[str, t.Any]) -> Metadata: ) def to_dict(self) -> dict[str, t.Any]: - """Convert metadata to a JSON-encodeable dictionary. + """Convert metadata to a JSON-encodable dictionary. Returns: Metadata object. diff --git a/singer_sdk/authenticators.py b/singer_sdk/authenticators.py index 61382daba..fcba67e7b 100644 --- a/singer_sdk/authenticators.py +++ b/singer_sdk/authenticators.py @@ -5,7 +5,7 @@ import base64 import math import typing as t -from datetime import datetime, timedelta +from datetime import timedelta from types import MappingProxyType from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit @@ -19,6 +19,8 @@ if t.TYPE_CHECKING: import logging + from pendulum import DateTime + from singer_sdk.streams.rest import RESTStream @@ -378,7 +380,7 @@ def __init__( # Initialize internal tracking attributes self.access_token: str | None = None self.refresh_token: str | None = None - self.last_refreshed: datetime | None = None + self.last_refreshed: DateTime | None = None self.expires_in: int | None = None @property @@ -462,9 +464,7 @@ def client_id(self) -> str | None: Returns: Optional client secret from stream config if it has been set. """ - if self.config: - return self.config.get("client_id") - return None + return self.config.get("client_id") if self.config else None @property def client_secret(self) -> str | None: @@ -473,9 +473,7 @@ def client_secret(self) -> str | None: Returns: Optional client secret from stream config if it has been set. """ - if self.config: - return self.config.get("client_secret") - return None + return self.config.get("client_secret") if self.config else None def is_token_valid(self) -> bool: """Check if token is valid. @@ -487,9 +485,7 @@ def is_token_valid(self) -> bool: return False if not self.expires_in: return True - if self.expires_in > (utc_now() - self.last_refreshed).total_seconds(): - return True - return False + return self.expires_in > (utc_now() - self.last_refreshed).total_seconds() # type: ignore[no-any-return] # Authentication and refresh def update_access_token(self) -> None: @@ -520,7 +516,7 @@ def update_access_token(self) -> None: self.expires_in = int(expiration) if expiration else None if self.expires_in is None: self.logger.debug( - "No expires_in receied in OAuth response and no " + "No expires_in received in OAuth response and no " "default_expiration set. Token will be treated as if it never " "expires.", ) @@ -566,7 +562,7 @@ def oauth_request_body(self) -> dict: @property def oauth_request_payload(self) -> dict: - """Return request paytload for OAuth request. + """Return request payload for OAuth request. Returns: Payload object for OAuth. diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index c6f957589..91846d46a 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -902,9 +902,6 @@ def merge_sql_types( if issubclass( generic_type, (sqlalchemy.types.String, sqlalchemy.types.Unicode), - ) or issubclass( - generic_type, - (sqlalchemy.types.String, sqlalchemy.types.Unicode), ): # If length None or 0 then is varchar max ? if ( diff --git a/singer_sdk/helpers/_conformers.py b/singer_sdk/helpers/_conformers.py index 46963284e..0ca70e85c 100644 --- a/singer_sdk/helpers/_conformers.py +++ b/singer_sdk/helpers/_conformers.py @@ -16,11 +16,13 @@ def snakecase(string: str) -> str: """ string = re.sub(r"[\-\.\s]", "_", string) string = ( - string[0].lower() - + re.sub( - r"[A-Z]", - lambda matched: "_" + str(matched.group(0).lower()), - string[1:], + ( + string[0].lower() + + re.sub( + r"[A-Z]", + lambda matched: f"_{matched.group(0).lower()!s}", + string[1:], + ) ) if string else string diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index c0abcb6b2..eeb244277 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -465,12 +465,9 @@ def _should_jsondump_value(key: str, value: t.Any, flattened_schema=None) -> boo if isinstance(value, (dict, list)): return True - if ( + return bool( flattened_schema and key in flattened_schema and "type" in flattened_schema[key] and set(flattened_schema[key]["type"]) == {"null", "object", "array"} - ): - return True - - return False + ) diff --git a/singer_sdk/helpers/_state.py b/singer_sdk/helpers/_state.py index 9d0102186..a42a38433 100644 --- a/singer_sdk/helpers/_state.py +++ b/singer_sdk/helpers/_state.py @@ -18,7 +18,7 @@ STARTING_MARKER = "starting_replication_value" -def get_state_if_exists( # noqa: PLR0911 +def get_state_if_exists( tap_state: dict, tap_stream_id: str, state_partition_context: dict | None = None, @@ -47,9 +47,7 @@ def get_state_if_exists( # noqa: PLR0911 stream_state = tap_state["bookmarks"][tap_stream_id] if not state_partition_context: - if key: - return stream_state.get(key, None) - return stream_state + return stream_state.get(key, None) if key else stream_state if "partitions" not in stream_state: return None # No partitions defined @@ -59,9 +57,7 @@ def get_state_if_exists( # noqa: PLR0911 ) if matched_partition is None: return None # Partition definition not present - if key: - return matched_partition.get(key, None) - return matched_partition + return matched_partition.get(key, None) if key else matched_partition def get_state_partitions_list(tap_state: dict, tap_stream_id: str) -> list[dict] | None: @@ -84,10 +80,7 @@ def _find_in_partitions_list( f"{{state_partition_context}}.\nMatching state values were: {found!s}" ) raise ValueError(msg) - if found: - return t.cast(dict, found[0]) - - return None + return t.cast(dict, found[0]) if found else None def _create_in_partitions_list( diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 6eb7b3879..3a87ab4b9 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -79,7 +79,7 @@ def is_secret_type(type_dict: dict) -> bool: """Return True if JSON Schema type definition appears to be a secret. Will return true if either `writeOnly` or `secret` are true on this type - or any of the type's subproperties. + or any of the type's sub-properties. Args: type_dict: The JSON Schema type to check. @@ -96,7 +96,7 @@ def is_secret_type(type_dict: dict) -> bool: return True if "properties" in type_dict: - # Recursively check subproperties and return True if any child is secret. + # Recursively check sub-properties and return True if any child is secret. return any( is_secret_type(child_type_dict) for child_type_dict in type_dict["properties"].values() @@ -388,6 +388,7 @@ def conform_record_data_types( return rec +# TODO: This is in dire need of refactoring. It's a mess. def _conform_record_data_types( # noqa: PLR0912 input_object: dict[str, t.Any], schema: dict, @@ -405,7 +406,7 @@ def _conform_record_data_types( # noqa: PLR0912 input_object: A single record schema: JSON schema the given input_object is expected to meet level: Specifies how recursive the conformance process should be - parent: '.' seperated path to this element from the object root (for logging) + parent: '.' separated path to this element from the object root (for logging) """ output_object: dict[str, t.Any] = {} unmapped_properties: list[str] = [] diff --git a/singer_sdk/io_base.py b/singer_sdk/io_base.py index 07da6e63e..d389bfd08 100644 --- a/singer_sdk/io_base.py +++ b/singer_sdk/io_base.py @@ -48,7 +48,7 @@ def _assert_line_requires(line_dict: dict, requires: set[str]) -> None: if not requires.issubset(line_dict): missing = requires - set(line_dict) msg = f"Line is missing required {', '.join(missing)} key(s): {line_dict}" - raise Exception(msg) + raise Exception(msg) # TODO: Raise a more specific exception def deserialize_json(self, line: str) -> dict: """Deserialize a line of json. diff --git a/singer_sdk/mapper.py b/singer_sdk/mapper.py index 5b19f0b07..b48c49727 100644 --- a/singer_sdk/mapper.py +++ b/singer_sdk/mapper.py @@ -273,10 +273,7 @@ def transform(self, record: dict) -> dict | None: The transformed record. """ transformed_record = self._transform_fn(record) - if not transformed_record: - return None - - return super().transform(transformed_record) + return super().transform(transformed_record) if transformed_record else None def get_filter_result(self, record: dict) -> bool: """Return True to include or False to exclude. @@ -291,7 +288,7 @@ def get_filter_result(self, record: dict) -> bool: @property def functions(self) -> dict[str, t.Callable]: - """Get availabale transformation functions. + """Get available transformation functions. Returns: Functions which should be available for expression evaluation. diff --git a/singer_sdk/metrics.py b/singer_sdk/metrics.py index 89d51a338..e4191eadf 100644 --- a/singer_sdk/metrics.py +++ b/singer_sdk/metrics.py @@ -263,10 +263,9 @@ def __exit__( exc_tb: The exception traceback. """ if Tag.STATUS not in self.tags: - if exc_type is None: - self.tags[Tag.STATUS] = Status.SUCCEEDED - else: - self.tags[Tag.STATUS] = Status.FAILED + self.tags[Tag.STATUS] = ( + Status.SUCCEEDED if exc_type is None else Status.FAILED + ) log(self.logger, Point("timer", self.metric, self.elapsed(), self.tags)) def elapsed(self) -> float: diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index ec4664f67..e56d5aab5 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -247,7 +247,7 @@ def _add_sdc_metadata_to_record( tz=datetime.timezone.utc, ).isoformat() record["_sdc_batched_at"] = ( - context.get("batch_start_time", None) + context.get("batch_start_time") or datetime.datetime.now(tz=datetime.timezone.utc) ).isoformat() record["_sdc_deleted_at"] = record.get("_sdc_deleted_at") diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 238e83dec..5c143818c 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -98,13 +98,7 @@ def schema_name(self) -> str | None: if default_target_schema: return default_target_schema - if len(parts) in {2, 3}: - # Stream name is a two-part or three-part identifier. - # Use the second-to-last part as the schema name. - return self.conform_name(parts[-2], "schema") - - # Schema name not detected. - return None + return self.conform_name(parts[-2], "schema") if len(parts) in {2, 3} else None @property def database_name(self) -> str | None: diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index d4b5a8dc3..f7a31e73c 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -121,10 +121,10 @@ def streams(self) -> dict[str, Stream]: Returns: A mapping of names to streams, using discovery or a provided catalog. """ - input_catalog = self.input_catalog - if self._streams is None: self._streams = {} + input_catalog = self.input_catalog + for stream in self.load_streams(): if input_catalog is not None: stream.apply_catalog(input_catalog) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index aabd8a640..c9d5d62ef 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -370,7 +370,7 @@ def _process_schema_message(self, message_dict: dict) -> None: stream_name = message_dict["stream"] schema = message_dict["schema"] - key_properties = message_dict.get("key_properties", None) + key_properties = message_dict.get("key_properties") do_registration = False if stream_name not in self.mapper.stream_maps: do_registration = True diff --git a/singer_sdk/testing/factory.py b/singer_sdk/testing/factory.py index ce6d2ec4c..f31930017 100644 --- a/singer_sdk/testing/factory.py +++ b/singer_sdk/testing/factory.py @@ -144,6 +144,7 @@ def runner(self) -> TapTestRunner | TargetTestRunner: return TapTestClass + # TODO: Refactor this. It's too long and nested. def _annotate_test_class( # noqa: C901 self, empty_test_class: type[BaseTestClass], diff --git a/singer_sdk/testing/legacy.py b/singer_sdk/testing/legacy.py index 5baa94034..5329b3224 100644 --- a/singer_sdk/testing/legacy.py +++ b/singer_sdk/testing/legacy.py @@ -150,10 +150,7 @@ def _get_tap_catalog( # Test discovery tap.run_discovery() catalog_dict = tap.catalog_dict - if select_all: - return _select_all(catalog_dict) - - return catalog_dict + return _select_all(catalog_dict) if select_all else catalog_dict def _select_all(catalog_dict: dict) -> dict: diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index a9c48303e..37c1f40c1 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -965,12 +965,14 @@ def to_jsonschema_type( msg = "Expected `str` or a SQLAlchemy `TypeEngine` object or type." raise ValueError(msg) - # Look for the type name within the known SQL type names: - for sqltype, jsonschema_type in sqltype_lookup.items(): - if sqltype.lower() in type_name.lower(): - return jsonschema_type - - return sqltype_lookup["string"] # safe failover to str + return next( + ( + jsonschema_type + for sqltype, jsonschema_type in sqltype_lookup.items() + if sqltype.lower() in type_name.lower() + ), + sqltype_lookup["string"], # safe failover to str + ) def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> bool: @@ -981,7 +983,7 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo type_check: A tuple of type strings to look for. Returns: - True if the schema suports the type. + True if the schema supports the type. """ if "type" in jsonschema_type: if isinstance(jsonschema_type["type"], (list, tuple)): @@ -991,12 +993,9 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo elif jsonschema_type.get("type") in type_check: return True - if any( + return any( _jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ()) - ): - return True - - return False + ) def to_sql_type( # noqa: PLR0911, C901 From 2a64cf612af63cec99efacf5aa3942bc685fdb46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:11:20 -0600 Subject: [PATCH 4/5] feat: Better error messages when config validation fails (#768) --- samples/sample_tap_sqlite/__init__.py | 1 + samples/sample_target_sqlite/__init__.py | 1 + singer_sdk/exceptions.py | 15 +++ singer_sdk/plugin_base.py | 72 ++++++++----- tests/core/conftest.py | 101 ++++++++++++++++++ tests/core/test_mapper_class.py | 54 ++++++++++ tests/core/test_streams.py | 127 ++++------------------- tests/core/test_tap_class.py | 92 ++++++++++++++++ tests/core/test_target_class.py | 54 ++++++++++ tests/samples/test_target_sqlite.py | 2 +- 10 files changed, 387 insertions(+), 132 deletions(-) create mode 100644 tests/core/conftest.py create mode 100644 tests/core/test_mapper_class.py create mode 100644 tests/core/test_tap_class.py create mode 100644 tests/core/test_target_class.py diff --git a/samples/sample_tap_sqlite/__init__.py b/samples/sample_tap_sqlite/__init__.py index 49b4365f0..3aed5d21d 100644 --- a/samples/sample_tap_sqlite/__init__.py +++ b/samples/sample_tap_sqlite/__init__.py @@ -48,6 +48,7 @@ class SQLiteTap(SQLTap): DB_PATH_CONFIG, th.StringType, description="The path to your SQLite database file(s).", + required=True, examples=["./path/to/my.db", "/absolute/path/to/my.db"], ), ).to_dict() diff --git a/samples/sample_target_sqlite/__init__.py b/samples/sample_target_sqlite/__init__.py index 40384facf..8e43a5e87 100644 --- a/samples/sample_target_sqlite/__init__.py +++ b/samples/sample_target_sqlite/__init__.py @@ -52,6 +52,7 @@ class SQLiteTarget(SQLTarget): DB_PATH_CONFIG, th.StringType, description="The path to your SQLite database file(s).", + required=True, ), ).to_dict() diff --git a/singer_sdk/exceptions.py b/singer_sdk/exceptions.py index 351776291..75135e800 100644 --- a/singer_sdk/exceptions.py +++ b/singer_sdk/exceptions.py @@ -12,6 +12,21 @@ class ConfigValidationError(Exception): """Raised when a user's config settings fail validation.""" + def __init__( + self, + message: str, + *, + errors: list[str] | None = None, + ) -> None: + """Initialize a ConfigValidationError. + + Args: + message: A message describing the error. + errors: A list of errors which caused the validation error. + """ + super().__init__(message) + self.errors = errors or [] + class FatalAPIError(Exception): """Exception raised when a failed request should not be considered retriable.""" diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 53e2cd2f2..b4e82296b 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -72,6 +72,43 @@ def __init__(self) -> None: super().__init__("Mapper not initialized. Please call setup_mapper() first.") +class SingerCommand(click.Command): + """Custom click command class for Singer packages.""" + + def __init__( + self, + *args: t.Any, + logger: logging.Logger, + **kwargs: t.Any, + ) -> None: + """Initialize the command. + + Args: + *args: Positional `click.Command` arguments. + logger: A logger instance. + **kwargs: Keyword `click.Command` arguments. + """ + super().__init__(*args, **kwargs) + self.logger = logger + + def invoke(self, ctx: click.Context) -> t.Any: # noqa: ANN401 + """Invoke the command, capturing warnings and logging them. + + Args: + ctx: The `click` context. + + Returns: + The result of the command invocation. + """ + logging.captureWarnings(capture=True) + try: + return super().invoke(ctx) + except ConfigValidationError as exc: + for error in exc.errors: + self.logger.error("Config validation error: %s", error) + sys.exit(1) + + class PluginBase(metaclass=abc.ABCMeta): """Abstract base class for taps.""" @@ -150,12 +187,12 @@ def __init__( if self._is_secret_config(k): config_dict[k] = SecretString(v) self._config = config_dict - self._validate_config(raise_errors=validate_config) - self._mapper: PluginMapper | None = None - metrics._setup_logging(self.config) self.metrics_logger = metrics.get_metrics_logger() + self._validate_config(raise_errors=validate_config) + self._mapper: PluginMapper | None = None + # Initialization timestamp self.__initialized_at = int(time.time() * 1000) @@ -351,27 +388,19 @@ def _is_secret_config(config_key: str) -> bool: """ return is_common_secret_key(config_key) - def _validate_config( - self, - *, - raise_errors: bool = True, - warnings_as_errors: bool = False, - ) -> tuple[list[str], list[str]]: + def _validate_config(self, *, raise_errors: bool = True) -> list[str]: """Validate configuration input against the plugin configuration JSON schema. Args: raise_errors: Flag to throw an exception if any validation errors are found. - warnings_as_errors: Flag to throw an exception if any warnings were emitted. Returns: - A tuple of configuration validation warnings and errors. + A list of validation errors. Raises: ConfigValidationError: If raise_errors is True and validation fails. """ - warnings: list[str] = [] errors: list[str] = [] - log_fn = self.logger.info config_jsonschema = self.config_jsonschema if config_jsonschema: @@ -389,19 +418,11 @@ def _validate_config( f"JSONSchema was: {config_jsonschema}" ) if raise_errors: - raise ConfigValidationError(summary) + raise ConfigValidationError(summary, errors=errors) - log_fn = self.logger.warning - else: - summary = f"Config validation passed with {len(warnings)} warnings." - for warning in warnings: - summary += f"\n{warning}" + self.logger.warning(summary) - if warnings_as_errors and raise_errors and warnings: - msg = f"One or more warnings ocurred during validation: {warnings}" - raise ConfigValidationError(msg) - log_fn(summary) - return warnings, errors + return errors @classmethod def print_version( @@ -555,7 +576,7 @@ def get_singer_command(cls: type[PluginBase]) -> click.Command: Returns: A callable CLI object. """ - return click.Command( + return SingerCommand( name=cls.name, callback=cls.invoke, context_settings={"help_option_names": ["--help"]}, @@ -596,6 +617,7 @@ def get_singer_command(cls: type[PluginBase]) -> click.Command: is_eager=True, ), ], + logger=cls.logger, ) @plugin_cli diff --git a/tests/core/conftest.py b/tests/core/conftest.py new file mode 100644 index 000000000..06355ccfe --- /dev/null +++ b/tests/core/conftest.py @@ -0,0 +1,101 @@ +"""Tap, target and stream test fixtures.""" + +from __future__ import annotations + +import typing as t + +import pendulum +import pytest + +from singer_sdk import Stream, Tap +from singer_sdk.typing import ( + DateTimeType, + IntegerType, + PropertiesList, + Property, + StringType, +) + + +class SimpleTestStream(Stream): + """Test stream class.""" + + name = "test" + schema = PropertiesList( + Property("id", IntegerType, required=True), + Property("value", StringType, required=True), + Property("updatedAt", DateTimeType, required=True), + ).to_dict() + replication_key = "updatedAt" + + def __init__(self, tap: Tap): + """Create a new stream.""" + super().__init__(tap, schema=self.schema, name=self.name) + + def get_records( + self, + context: dict | None, # noqa: ARG002 + ) -> t.Iterable[dict[str, t.Any]]: + """Generate records.""" + yield {"id": 1, "value": "Egypt"} + yield {"id": 2, "value": "Germany"} + yield {"id": 3, "value": "India"} + + +class UnixTimestampIncrementalStream(SimpleTestStream): + name = "unix_ts" + schema = PropertiesList( + Property("id", IntegerType, required=True), + Property("value", StringType, required=True), + Property("updatedAt", IntegerType, required=True), + ).to_dict() + replication_key = "updatedAt" + + +class UnixTimestampIncrementalStream2(UnixTimestampIncrementalStream): + name = "unix_ts_override" + + def compare_start_date(self, value: str, start_date_value: str) -> str: + """Compare a value to a start date value.""" + + start_timestamp = pendulum.parse(start_date_value).format("X") + return max(value, start_timestamp, key=float) + + +class SimpleTestTap(Tap): + """Test tap class.""" + + name = "test-tap" + config_jsonschema = PropertiesList( + Property("username", StringType, required=True), + Property("password", StringType, required=True), + Property("start_date", DateTimeType), + additional_properties=False, + ).to_dict() + + def discover_streams(self) -> list[Stream]: + """List all streams.""" + return [ + SimpleTestStream(self), + UnixTimestampIncrementalStream(self), + UnixTimestampIncrementalStream2(self), + ] + + +@pytest.fixture +def tap_class(): + """Return the tap class.""" + return SimpleTestTap + + +@pytest.fixture +def tap() -> SimpleTestTap: + """Tap instance.""" + return SimpleTestTap( + config={ + "username": "utest", + "password": "ptest", + "start_date": "2021-01-01", + }, + parse_env_config=False, + ) diff --git a/tests/core/test_mapper_class.py b/tests/core/test_mapper_class.py new file mode 100644 index 000000000..0f0c1192a --- /dev/null +++ b/tests/core/test_mapper_class.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +from contextlib import nullcontext + +import pytest +from click.testing import CliRunner + +from samples.sample_mapper.mapper import StreamTransform +from singer_sdk.exceptions import ConfigValidationError + + +@pytest.mark.parametrize( + "config_dict,expectation,errors", + [ + pytest.param( + {}, + pytest.raises(ConfigValidationError, match="Config validation failed"), + ["'stream_maps' is a required property"], + id="missing_stream_maps", + ), + pytest.param( + {"stream_maps": {}}, + nullcontext(), + [], + id="valid_config", + ), + ], +) +def test_config_errors(config_dict: dict, expectation, errors: list[str]): + with expectation as exc: + StreamTransform(config=config_dict, validate_config=True) + + if isinstance(exc, pytest.ExceptionInfo): + assert exc.value.errors == errors + + +def test_cli_help(): + """Test the CLI help message.""" + runner = CliRunner(mix_stderr=False) + result = runner.invoke(StreamTransform.cli, ["--help"]) + assert result.exit_code == 0 + assert "Show this message and exit." in result.output + + +def test_cli_config_validation(tmp_path): + """Test the CLI config validation.""" + runner = CliRunner(mix_stderr=False) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({})) + result = runner.invoke(StreamTransform.cli, ["--config", str(config_path)]) + assert result.exit_code == 1 + assert not result.stdout + assert "'stream_maps' is a required property" in result.stderr diff --git a/tests/core/test_streams.py b/tests/core/test_streams.py index a3a451086..8a415e55d 100644 --- a/tests/core/test_streams.py +++ b/tests/core/test_streams.py @@ -16,68 +16,17 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers.jsonpath import _compile_jsonpath, extract_jsonpath from singer_sdk.pagination import first -from singer_sdk.streams.core import ( - REPLICATION_FULL_TABLE, - REPLICATION_INCREMENTAL, - Stream, -) +from singer_sdk.streams.core import REPLICATION_FULL_TABLE, REPLICATION_INCREMENTAL from singer_sdk.streams.graphql import GraphQLStream from singer_sdk.streams.rest import RESTStream -from singer_sdk.tap_base import Tap -from singer_sdk.typing import ( - DateTimeType, - IntegerType, - PropertiesList, - Property, - StringType, -) +from singer_sdk.typing import IntegerType, PropertiesList, Property, StringType +from tests.core.conftest import SimpleTestStream CONFIG_START_DATE = "2021-01-01" - -class SimpleTestStream(Stream): - """Test stream class.""" - - name = "test" - schema = PropertiesList( - Property("id", IntegerType, required=True), - Property("value", StringType, required=True), - Property("updatedAt", DateTimeType, required=True), - ).to_dict() - replication_key = "updatedAt" - - def __init__(self, tap: Tap): - """Create a new stream.""" - super().__init__(tap, schema=self.schema, name=self.name) - - def get_records( - self, - context: dict | None, # noqa: ARG002 - ) -> t.Iterable[dict[str, t.Any]]: - """Generate records.""" - yield {"id": 1, "value": "Egypt"} - yield {"id": 2, "value": "Germany"} - yield {"id": 3, "value": "India"} - - -class UnixTimestampIncrementalStream(SimpleTestStream): - name = "unix_ts" - schema = PropertiesList( - Property("id", IntegerType, required=True), - Property("value", StringType, required=True), - Property("updatedAt", IntegerType, required=True), - ).to_dict() - replication_key = "updatedAt" - - -class UnixTimestampIncrementalStream2(UnixTimestampIncrementalStream): - name = "unix_ts_override" - - def compare_start_date(self, value: str, start_date_value: str) -> str: - """Compare a value to a start date value.""" - - start_timestamp = pendulum.parse(start_date_value).format("X") - return max(value, start_timestamp, key=float) +if t.TYPE_CHECKING: + from singer_sdk import Stream, Tap + from tests.core.conftest import SimpleTestTap class RestTestStream(RESTStream): @@ -124,43 +73,13 @@ class GraphqlTestStream(GraphQLStream): replication_key = "updatedAt" -class SimpleTestTap(Tap): - """Test tap class.""" - - name = "test-tap" - settings_jsonschema = PropertiesList(Property("start_date", DateTimeType)).to_dict() - - def discover_streams(self) -> list[Stream]: - """List all streams.""" - return [ - SimpleTestStream(self), - UnixTimestampIncrementalStream(self), - UnixTimestampIncrementalStream2(self), - ] - - @pytest.fixture -def tap() -> SimpleTestTap: - """Tap instance.""" - return SimpleTestTap( - config={"start_date": CONFIG_START_DATE}, - parse_env_config=False, - ) - - -@pytest.fixture -def stream(tap: SimpleTestTap) -> SimpleTestStream: - """Create a new stream instance.""" - return t.cast(SimpleTestStream, tap.load_streams()[0]) - - -@pytest.fixture -def unix_timestamp_stream(tap: SimpleTestTap) -> UnixTimestampIncrementalStream: +def stream(tap): """Create a new stream instance.""" - return t.cast(UnixTimestampIncrementalStream, tap.load_streams()[1]) + return tap.load_streams()[0] -def test_stream_apply_catalog(stream: SimpleTestStream): +def test_stream_apply_catalog(stream: Stream): """Applying a catalog to a stream should overwrite fields.""" assert stream.primary_keys == [] assert stream.replication_key == "updatedAt" @@ -251,7 +170,7 @@ def test_stream_apply_catalog(stream: SimpleTestStream): ], ) def test_stream_starting_timestamp( - tap: SimpleTestTap, + tap: Tap, stream_name: str, bookmark_value: str, expected_starting_value: t.Any, @@ -353,12 +272,7 @@ class InvalidReplicationKeyStream(SimpleTestStream): "nested_values", ], ) -def test_jsonpath_rest_stream( - tap: SimpleTestTap, - path: str, - content: str, - result: list[dict], -): +def test_jsonpath_rest_stream(tap: Tap, path: str, content: str, result: list[dict]): """Validate records are extracted correctly from the API response.""" fake_response = requests.Response() fake_response._content = str.encode(content) @@ -371,7 +285,7 @@ def test_jsonpath_rest_stream( assert list(records) == result -def test_jsonpath_graphql_stream_default(tap: SimpleTestTap): +def test_jsonpath_graphql_stream_default(tap: Tap): """Validate graphql JSONPath, defaults to the stream name.""" content = """{ "data": { @@ -391,7 +305,7 @@ def test_jsonpath_graphql_stream_default(tap: SimpleTestTap): assert list(records) == [{"id": 1, "value": "abc"}, {"id": 2, "value": "def"}] -def test_jsonpath_graphql_stream_override(tap: SimpleTestTap): +def test_jsonpath_graphql_stream_override(tap: Tap): """Validate graphql jsonpath can be updated.""" content = """[ {"id": 1, "value": "abc"}, @@ -478,7 +392,7 @@ def records_jsonpath(cls): # noqa: N805 ], ) def test_next_page_token_jsonpath( - tap: SimpleTestTap, + tap: Tap, path: str, content: str, headers: dict, @@ -510,7 +424,7 @@ def test_cached_jsonpath(): assert recompiled is compiled -def test_sync_costs_calculation(tap: SimpleTestTap, caplog): +def test_sync_costs_calculation(tap: Tap, caplog): """Test sync costs are added up correctly.""" fake_request = requests.PreparedRequest() fake_response = requests.Response() @@ -595,7 +509,7 @@ def calculate_test_cost( ), ], ) -def test_stream_class_selection(input_catalog, selection): +def test_stream_class_selection(tap_class, input_catalog, selection): """Test stream class selection.""" class SelectedStream(RESTStream): @@ -607,11 +521,12 @@ class UnselectedStream(SelectedStream): name = "unselected_stream" selected_by_default = False - class MyTap(SimpleTestTap): + class MyTap(tap_class): def discover_streams(self): return [SelectedStream(self), UnselectedStream(self)] # Check that the selected stream is selected - tap = MyTap(config=None, catalog=input_catalog) - for stream in selection: - assert tap.streams[stream].selected is selection[stream] + tap = MyTap(config=None, catalog=input_catalog, validate_config=False) + assert all( + tap.streams[stream].selected is selection[stream] for stream in selection + ) diff --git a/tests/core/test_tap_class.py b/tests/core/test_tap_class.py new file mode 100644 index 000000000..93015fbb1 --- /dev/null +++ b/tests/core/test_tap_class.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +import typing as t +from contextlib import nullcontext + +import pytest +from click.testing import CliRunner + +from singer_sdk.exceptions import ConfigValidationError + +if t.TYPE_CHECKING: + from singer_sdk import Tap + + +@pytest.mark.parametrize( + "config_dict,expectation,errors", + [ + pytest.param( + {}, + pytest.raises(ConfigValidationError, match="Config validation failed"), + ["'username' is a required property", "'password' is a required property"], + id="missing_username_and_password", + ), + pytest.param( + {"username": "utest"}, + pytest.raises(ConfigValidationError, match="Config validation failed"), + ["'password' is a required property"], + id="missing_password", + ), + pytest.param( + {"username": "utest", "password": "ptest", "extra": "not valid"}, + pytest.raises(ConfigValidationError, match="Config validation failed"), + ["Additional properties are not allowed ('extra' was unexpected)"], + id="extra_property", + ), + pytest.param( + {"username": "utest", "password": "ptest"}, + nullcontext(), + [], + id="valid_config", + ), + ], +) +def test_config_errors( + tap_class: type[Tap], + config_dict: dict, + expectation, + errors: list[str], +): + with expectation as exc: + tap_class(config=config_dict, validate_config=True) + + if isinstance(exc, pytest.ExceptionInfo): + assert exc.value.errors == errors + + +def test_cli(tap_class: type[Tap]): + """Test the CLI.""" + runner = CliRunner(mix_stderr=False) + result = runner.invoke(tap_class.cli, ["--help"]) + assert result.exit_code == 0 + assert "Show this message and exit." in result.output + + +def test_cli_config_validation(tap_class: type[Tap], tmp_path): + """Test the CLI config validation.""" + runner = CliRunner(mix_stderr=False) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({})) + result = runner.invoke(tap_class.cli, ["--config", str(config_path)]) + assert result.exit_code == 1 + assert not result.stdout + assert "'username' is a required property" in result.stderr + assert "'password' is a required property" in result.stderr + + +def test_cli_discover(tap_class: type[Tap], tmp_path): + """Test the CLI discover command.""" + runner = CliRunner(mix_stderr=False) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({})) + result = runner.invoke( + tap_class.cli, + [ + "--config", + str(config_path), + "--discover", + ], + ) + assert result.exit_code == 0 + assert "streams" in json.loads(result.stdout) diff --git a/tests/core/test_target_class.py b/tests/core/test_target_class.py new file mode 100644 index 000000000..f84ae1dae --- /dev/null +++ b/tests/core/test_target_class.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +from contextlib import nullcontext + +import pytest +from click.testing import CliRunner + +from samples.sample_target_sqlite import SQLiteTarget +from singer_sdk.exceptions import ConfigValidationError + + +@pytest.mark.parametrize( + "config_dict,expectation,errors", + [ + pytest.param( + {}, + pytest.raises(ConfigValidationError, match="Config validation failed"), + ["'path_to_db' is a required property"], + id="missing_path_to_db", + ), + pytest.param( + {"path_to_db": "sqlite://test.db"}, + nullcontext(), + [], + id="valid_config", + ), + ], +) +def test_config_errors(config_dict: dict, expectation, errors: list[str]): + with expectation as exc: + SQLiteTarget(config=config_dict, validate_config=True) + + if isinstance(exc, pytest.ExceptionInfo): + assert exc.value.errors == errors + + +def test_cli(): + """Test the CLI.""" + runner = CliRunner(mix_stderr=False) + result = runner.invoke(SQLiteTarget.cli, ["--help"]) + assert result.exit_code == 0 + assert "Show this message and exit." in result.output + + +def test_cli_config_validation(tmp_path): + """Test the CLI config validation.""" + runner = CliRunner(mix_stderr=False) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({})) + result = runner.invoke(SQLiteTarget.cli, ["--config", str(config_path)]) + assert result.exit_code == 1 + assert not result.stdout + assert "'path_to_db' is a required property" in result.stderr diff --git a/tests/samples/test_target_sqlite.py b/tests/samples/test_target_sqlite.py index 727b760ba..a66805a09 100644 --- a/tests/samples/test_target_sqlite.py +++ b/tests/samples/test_target_sqlite.py @@ -36,7 +36,7 @@ def path_to_target_db(tmp_path: Path) -> Path: @pytest.fixture -def sqlite_target_test_config(path_to_target_db: str) -> dict: +def sqlite_target_test_config(path_to_target_db: Path) -> dict: """Get configuration dictionary for target-csv.""" return {"path_to_db": str(path_to_target_db)} From 5b849128a28689087b5d8bfa8406ce615ed92eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:00:15 -0600 Subject: [PATCH 5/5] ci: Explicitly install `poetry-plugin-export` in CI environments that need to export requirements (#2055) * ci: Explicitly install `poetry-plugin-export` in CI environments that need exporting requirements * ci: BUmp `poetry-plugin-export` --- .github/workflows/constraints.txt | 1 + .github/workflows/cookiecutter-e2e.yml | 8 ++++++-- .github/workflows/test.yml | 22 +++++++++++++++------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index 8ee1d08be..0e226fe7b 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,5 +1,6 @@ pip==23.3.1 poetry==1.7.1 +poetry-plugin-export==1.6.0 pre-commit==3.5.0 nox==2023.4.22 nox-poetry==1.0.3 diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index d3ed5cf9f..b36df1fa4 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -38,9 +38,13 @@ jobs: pip --version - name: Install Poetry + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | pipx install poetry + pipx inject poetry poetry-plugin-export poetry --version + poetry self show plugins - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v4.7.1 @@ -59,8 +63,8 @@ jobs: env: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | - pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox - pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry + pipx install nox + pipx inject nox nox-poetry nox --version - name: Run Nox diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 19381e495..7077ba5be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,9 @@ jobs: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | pipx install poetry + pipx inject poetry poetry-plugin-export poetry --version + poetry self show plugins - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v4.7.1 @@ -83,8 +85,8 @@ jobs: env: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | - pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox - pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry + pipx install nox + pipx inject nox nox-poetry nox --version - name: Run Nox @@ -122,7 +124,9 @@ jobs: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | pipx install poetry + pipx inject poetry poetry-plugin-export poetry --version + poetry self show plugins - name: Setup Python 3.10 uses: actions/setup-python@v4.7.1 @@ -143,8 +147,8 @@ jobs: env: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | - pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox - pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry + pipx install nox + pipx inject nox nox-poetry nox --version - name: Run Nox @@ -160,9 +164,13 @@ jobs: uses: actions/checkout@v4.1.1 - name: Install Poetry + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | - pipx install --pip-args=--constraint=.github/workflows/constraints.txt poetry + pipx install poetry + pipx inject poetry poetry-plugin-export poetry --version + poetry self show plugins - name: Set up Python uses: actions/setup-python@v4.7.1 @@ -185,8 +193,8 @@ jobs: env: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | - pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox - pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry + pipx install nox + pipx inject nox nox-poetry nox --version - name: Combine coverage data and display human readable report