diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index cc5ef94bc130..efe882a656d2 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -99,7 +99,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} git_branch: ${{ github.head_ref }} git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - github_token: ${{ github.token }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} subcommand: "test --modified" @@ -116,6 +116,6 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} git_branch: ${{ steps.extract_branch.outputs.branch }} git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - github_token: ${{ github.token }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} subcommand: "test ${{ inputs.airbyte_ci_subcommand}}" diff --git a/airbyte-ci/connectors/live-tests/README.md b/airbyte-ci/connectors/live-tests/README.md index 7da1b3ba332a..72aeb76f9bc8 100644 --- a/airbyte-ci/connectors/live-tests/README.md +++ b/airbyte-ci/connectors/live-tests/README.md @@ -237,6 +237,9 @@ The traffic recorded on the control connector is passed to the target connector ## Changelog +### 0.16.0 +Enable running with airbyte-ci. + ### 0.15.0 Automatic retrieval of connection objects for regression tests. The connection id is not required anymore. diff --git a/airbyte-ci/connectors/live-tests/poetry.lock b/airbyte-ci/connectors/live-tests/poetry.lock index 249c6541cf90..e9ef1278246b 100644 --- a/airbyte-ci/connectors/live-tests/poetry.lock +++ b/airbyte-ci/connectors/live-tests/poetry.lock @@ -299,13 +299,13 @@ files = [ [[package]] name = "beartype" -version = "0.18.3" +version = "0.18.5" description = "Unbearably fast runtime type checking in pure Python." optional = false python-versions = ">=3.8.0" files = [ - {file = "beartype-0.18.3-py3-none-any.whl", hash = "sha256:0131717230b04a7f1994422edaf237735febc90c0c98bd0094ce69e0a6706fff"}, - {file = "beartype-0.18.3.tar.gz", hash = "sha256:04853d78493f3c84b03956f1fa0c1e5b84c5332a760559ddc8f2cf6141ddf791"}, + {file = "beartype-0.18.5-py3-none-any.whl", hash = "sha256:5301a14f2a9a5540fe47ec6d34d758e9cd8331d36c4760fc7a5499ab86310089"}, + {file = "beartype-0.18.5.tar.gz", hash = "sha256:264ddc2f1da9ec94ff639141fbe33d22e12a9f75aa863b83b7046ffff1381927"}, ] [package.extras] @@ -720,8 +720,8 @@ tqdm = "^4.66.2" [package.source] type = "git" url = "git@github.com:airbytehq/airbyte-platform-internal" -reference = "augustin/04-16-connection_retriever_automatically_fetch_connections_for_testing" -resolved_reference = "e78ff1bed6dbc55dcd6a549fa212be566804bcaa" +reference = "HEAD" +resolved_reference = "d29ccdab38d63187066e5e8b9ae14e49d7ab697f" subdirectory = "tools/connection-retriever" [[package]] @@ -940,13 +940,13 @@ xmod = "*" [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, ] [package.extras] @@ -1176,13 +1176,13 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "google-cloud-bigquery" -version = "3.20.1" +version = "3.21.0" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.20.1.tar.gz", hash = "sha256:318aa3abab5f1900ee24f63ba8bd02b9cdafaa942d738b4dc14a4ef2cc2d925f"}, - {file = "google_cloud_bigquery-3.20.1-py2.py3-none-any.whl", hash = "sha256:d3e62fe61138c658b8853c402e2d8fb9346c84e602e21e3a26584be10fc5b0a4"}, + {file = "google-cloud-bigquery-3.21.0.tar.gz", hash = "sha256:6265c39f9d5bdf50f11cb81a9c2a0605d285df34ac139de0d2333b1250add0ff"}, + {file = "google_cloud_bigquery-3.21.0-py2.py3-none-any.whl", hash = "sha256:83a090aae16b3a687ef22e7b0a1b551e18da615b1c4855c5f312f198959e7739"}, ] [package.dependencies] @@ -1267,13 +1267,13 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4 [[package]] name = "google-cloud-secret-manager" -version = "2.19.0" +version = "2.20.0" description = "Google Cloud Secret Manager API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-secret-manager-2.19.0.tar.gz", hash = "sha256:bb918435835a14eb94785f4d4d9087bdcf1b6de306432d7edaa7d62e7f780c30"}, - {file = "google_cloud_secret_manager-2.19.0-py2.py3-none-any.whl", hash = "sha256:7dd9ad9ab3e70f9a7fbac432938b702ba23bce1207e9bda86463b6d6b1f5cdbb"}, + {file = "google-cloud-secret-manager-2.20.0.tar.gz", hash = "sha256:a086a7413aaf4fffbd1c4fe9229ef0ce9bcf48f5a8df5b449c4a32deb5a2cfde"}, + {file = "google_cloud_secret_manager-2.20.0-py2.py3-none-any.whl", hash = "sha256:c20bf22e59d220c51aa84a1db3411b14b83aa71f788fae8d273c03a4bf3e77ed"}, ] [package.dependencies] @@ -1527,84 +1527,84 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "grpcio" -version = "1.62.1" +version = "1.62.2" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.62.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:179bee6f5ed7b5f618844f760b6acf7e910988de77a4f75b95bbfaa8106f3c1e"}, - {file = "grpcio-1.62.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:48611e4fa010e823ba2de8fd3f77c1322dd60cb0d180dc6630a7e157b205f7ea"}, - {file = "grpcio-1.62.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:b2a0e71b0a2158aa4bce48be9f8f9eb45cbd17c78c7443616d00abbe2a509f6d"}, - {file = "grpcio-1.62.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbe80577c7880911d3ad65e5ecc997416c98f354efeba2f8d0f9112a67ed65a5"}, - {file = "grpcio-1.62.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f6c693d446964e3292425e1d16e21a97a48ba9172f2d0df9d7b640acb99243"}, - {file = "grpcio-1.62.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:77c339403db5a20ef4fed02e4d1a9a3d9866bf9c0afc77a42234677313ea22f3"}, - {file = "grpcio-1.62.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b5a4ea906db7dec694098435d84bf2854fe158eb3cd51e1107e571246d4d1d70"}, - {file = "grpcio-1.62.1-cp310-cp310-win32.whl", hash = "sha256:4187201a53f8561c015bc745b81a1b2d278967b8de35f3399b84b0695e281d5f"}, - {file = "grpcio-1.62.1-cp310-cp310-win_amd64.whl", hash = "sha256:844d1f3fb11bd1ed362d3fdc495d0770cfab75761836193af166fee113421d66"}, - {file = "grpcio-1.62.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:833379943d1728a005e44103f17ecd73d058d37d95783eb8f0b28ddc1f54d7b2"}, - {file = "grpcio-1.62.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:c7fcc6a32e7b7b58f5a7d27530669337a5d587d4066060bcb9dee7a8c833dfb7"}, - {file = "grpcio-1.62.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:fa7d28eb4d50b7cbe75bb8b45ed0da9a1dc5b219a0af59449676a29c2eed9698"}, - {file = "grpcio-1.62.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48f7135c3de2f298b833be8b4ae20cafe37091634e91f61f5a7eb3d61ec6f660"}, - {file = "grpcio-1.62.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71f11fd63365ade276c9d4a7b7df5c136f9030e3457107e1791b3737a9b9ed6a"}, - {file = "grpcio-1.62.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4b49fd8fe9f9ac23b78437da94c54aa7e9996fbb220bac024a67469ce5d0825f"}, - {file = "grpcio-1.62.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:482ae2ae78679ba9ed5752099b32e5fe580443b4f798e1b71df412abf43375db"}, - {file = "grpcio-1.62.1-cp311-cp311-win32.whl", hash = "sha256:1faa02530b6c7426404372515fe5ddf66e199c2ee613f88f025c6f3bd816450c"}, - {file = "grpcio-1.62.1-cp311-cp311-win_amd64.whl", hash = "sha256:5bd90b8c395f39bc82a5fb32a0173e220e3f401ff697840f4003e15b96d1befc"}, - {file = "grpcio-1.62.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:b134d5d71b4e0837fff574c00e49176051a1c532d26c052a1e43231f252d813b"}, - {file = "grpcio-1.62.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d1f6c96573dc09d50dbcbd91dbf71d5cf97640c9427c32584010fbbd4c0e0037"}, - {file = "grpcio-1.62.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:359f821d4578f80f41909b9ee9b76fb249a21035a061a327f91c953493782c31"}, - {file = "grpcio-1.62.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a485f0c2010c696be269184bdb5ae72781344cb4e60db976c59d84dd6354fac9"}, - {file = "grpcio-1.62.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b50b09b4dc01767163d67e1532f948264167cd27f49e9377e3556c3cba1268e1"}, - {file = "grpcio-1.62.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3227c667dccbe38f2c4d943238b887bac588d97c104815aecc62d2fd976e014b"}, - {file = "grpcio-1.62.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3952b581eb121324853ce2b191dae08badb75cd493cb4e0243368aa9e61cfd41"}, - {file = "grpcio-1.62.1-cp312-cp312-win32.whl", hash = "sha256:83a17b303425104d6329c10eb34bba186ffa67161e63fa6cdae7776ff76df73f"}, - {file = "grpcio-1.62.1-cp312-cp312-win_amd64.whl", hash = "sha256:6696ffe440333a19d8d128e88d440f91fb92c75a80ce4b44d55800e656a3ef1d"}, - {file = "grpcio-1.62.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e3393b0823f938253370ebef033c9fd23d27f3eae8eb9a8f6264900c7ea3fb5a"}, - {file = "grpcio-1.62.1-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:83e7ccb85a74beaeae2634f10eb858a0ed1a63081172649ff4261f929bacfd22"}, - {file = "grpcio-1.62.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:882020c87999d54667a284c7ddf065b359bd00251fcd70279ac486776dbf84ec"}, - {file = "grpcio-1.62.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a10383035e864f386fe096fed5c47d27a2bf7173c56a6e26cffaaa5a361addb1"}, - {file = "grpcio-1.62.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:960edebedc6b9ada1ef58e1c71156f28689978188cd8cff3b646b57288a927d9"}, - {file = "grpcio-1.62.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:23e2e04b83f347d0aadde0c9b616f4726c3d76db04b438fd3904b289a725267f"}, - {file = "grpcio-1.62.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978121758711916d34fe57c1f75b79cdfc73952f1481bb9583399331682d36f7"}, - {file = "grpcio-1.62.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9084086190cc6d628f282e5615f987288b95457292e969b9205e45b442276407"}, - {file = "grpcio-1.62.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:22bccdd7b23c420a27fd28540fb5dcbc97dc6be105f7698cb0e7d7a420d0e362"}, - {file = "grpcio-1.62.1-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:8999bf1b57172dbc7c3e4bb3c732658e918f5c333b2942243f10d0d653953ba9"}, - {file = "grpcio-1.62.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:d9e52558b8b8c2f4ac05ac86344a7417ccdd2b460a59616de49eb6933b07a0bd"}, - {file = "grpcio-1.62.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1714e7bc935780bc3de1b3fcbc7674209adf5208ff825799d579ffd6cd0bd505"}, - {file = "grpcio-1.62.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8842ccbd8c0e253c1f189088228f9b433f7a93b7196b9e5b6f87dba393f5d5d"}, - {file = "grpcio-1.62.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1f1e7b36bdff50103af95a80923bf1853f6823dd62f2d2a2524b66ed74103e49"}, - {file = "grpcio-1.62.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bba97b8e8883a8038606480d6b6772289f4c907f6ba780fa1f7b7da7dfd76f06"}, - {file = "grpcio-1.62.1-cp38-cp38-win32.whl", hash = "sha256:a7f615270fe534548112a74e790cd9d4f5509d744dd718cd442bf016626c22e4"}, - {file = "grpcio-1.62.1-cp38-cp38-win_amd64.whl", hash = "sha256:e6c8c8693df718c5ecbc7babb12c69a4e3677fd11de8886f05ab22d4e6b1c43b"}, - {file = "grpcio-1.62.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:73db2dc1b201d20ab7083e7041946910bb991e7e9761a0394bbc3c2632326483"}, - {file = "grpcio-1.62.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:407b26b7f7bbd4f4751dbc9767a1f0716f9fe72d3d7e96bb3ccfc4aace07c8de"}, - {file = "grpcio-1.62.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:f8de7c8cef9261a2d0a62edf2ccea3d741a523c6b8a6477a340a1f2e417658de"}, - {file = "grpcio-1.62.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd5c8a1af40ec305d001c60236308a67e25419003e9bb3ebfab5695a8d0b369"}, - {file = "grpcio-1.62.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be0477cb31da67846a33b1a75c611f88bfbcd427fe17701b6317aefceee1b96f"}, - {file = "grpcio-1.62.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:60dcd824df166ba266ee0cfaf35a31406cd16ef602b49f5d4dfb21f014b0dedd"}, - {file = "grpcio-1.62.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:973c49086cabab773525f6077f95e5a993bfc03ba8fc32e32f2c279497780585"}, - {file = "grpcio-1.62.1-cp39-cp39-win32.whl", hash = "sha256:12859468e8918d3bd243d213cd6fd6ab07208195dc140763c00dfe901ce1e1b4"}, - {file = "grpcio-1.62.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7209117bbeebdfa5d898205cc55153a51285757902dd73c47de498ad4d11332"}, - {file = "grpcio-1.62.1.tar.gz", hash = "sha256:6c455e008fa86d9e9a9d85bb76da4277c0d7d9668a3bfa70dbe86e9f3c759947"}, + {file = "grpcio-1.62.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:66344ea741124c38588a664237ac2fa16dfd226964cca23ddc96bd4accccbde5"}, + {file = "grpcio-1.62.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5dab7ac2c1e7cb6179c6bfad6b63174851102cbe0682294e6b1d6f0981ad7138"}, + {file = "grpcio-1.62.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:3ad00f3f0718894749d5a8bb0fa125a7980a2f49523731a9b1fabf2b3522aa43"}, + {file = "grpcio-1.62.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e72ddfee62430ea80133d2cbe788e0d06b12f865765cb24a40009668bd8ea05"}, + {file = "grpcio-1.62.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53d3a59a10af4c2558a8e563aed9f256259d2992ae0d3037817b2155f0341de1"}, + {file = "grpcio-1.62.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1511a303f8074f67af4119275b4f954189e8313541da7b88b1b3a71425cdb10"}, + {file = "grpcio-1.62.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b94d41b7412ef149743fbc3178e59d95228a7064c5ab4760ae82b562bdffb199"}, + {file = "grpcio-1.62.2-cp310-cp310-win32.whl", hash = "sha256:a75af2fc7cb1fe25785be7bed1ab18cef959a376cdae7c6870184307614caa3f"}, + {file = "grpcio-1.62.2-cp310-cp310-win_amd64.whl", hash = "sha256:80407bc007754f108dc2061e37480238b0dc1952c855e86a4fc283501ee6bb5d"}, + {file = "grpcio-1.62.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:c1624aa686d4b36790ed1c2e2306cc3498778dffaf7b8dd47066cf819028c3ad"}, + {file = "grpcio-1.62.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:1c1bb80299bdef33309dff03932264636450c8fdb142ea39f47e06a7153d3063"}, + {file = "grpcio-1.62.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:db068bbc9b1fa16479a82e1ecf172a93874540cb84be69f0b9cb9b7ac3c82670"}, + {file = "grpcio-1.62.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2cc8a308780edbe2c4913d6a49dbdb5befacdf72d489a368566be44cadaef1a"}, + {file = "grpcio-1.62.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0695ae31a89f1a8fc8256050329a91a9995b549a88619263a594ca31b76d756"}, + {file = "grpcio-1.62.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88b4f9ee77191dcdd8810241e89340a12cbe050be3e0d5f2f091c15571cd3930"}, + {file = "grpcio-1.62.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a0204532aa2f1afd467024b02b4069246320405bc18abec7babab03e2644e75"}, + {file = "grpcio-1.62.2-cp311-cp311-win32.whl", hash = "sha256:6e784f60e575a0de554ef9251cbc2ceb8790914fe324f11e28450047f264ee6f"}, + {file = "grpcio-1.62.2-cp311-cp311-win_amd64.whl", hash = "sha256:112eaa7865dd9e6d7c0556c8b04ae3c3a2dc35d62ad3373ab7f6a562d8199200"}, + {file = "grpcio-1.62.2-cp312-cp312-linux_armv7l.whl", hash = "sha256:65034473fc09628a02fb85f26e73885cf1ed39ebd9cf270247b38689ff5942c5"}, + {file = "grpcio-1.62.2-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d2c1771d0ee3cf72d69bb5e82c6a82f27fbd504c8c782575eddb7839729fbaad"}, + {file = "grpcio-1.62.2-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:3abe6838196da518863b5d549938ce3159d809218936851b395b09cad9b5d64a"}, + {file = "grpcio-1.62.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5ffeb269f10cedb4f33142b89a061acda9f672fd1357331dbfd043422c94e9e"}, + {file = "grpcio-1.62.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:404d3b4b6b142b99ba1cff0b2177d26b623101ea2ce51c25ef6e53d9d0d87bcc"}, + {file = "grpcio-1.62.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:262cda97efdabb20853d3b5a4c546a535347c14b64c017f628ca0cc7fa780cc6"}, + {file = "grpcio-1.62.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17708db5b11b966373e21519c4c73e5a750555f02fde82276ea2a267077c68ad"}, + {file = "grpcio-1.62.2-cp312-cp312-win32.whl", hash = "sha256:b7ec9e2f8ffc8436f6b642a10019fc513722858f295f7efc28de135d336ac189"}, + {file = "grpcio-1.62.2-cp312-cp312-win_amd64.whl", hash = "sha256:aa787b83a3cd5e482e5c79be030e2b4a122ecc6c5c6c4c42a023a2b581fdf17b"}, + {file = "grpcio-1.62.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:cfd23ad29bfa13fd4188433b0e250f84ec2c8ba66b14a9877e8bce05b524cf54"}, + {file = "grpcio-1.62.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:af15e9efa4d776dfcecd1d083f3ccfb04f876d613e90ef8432432efbeeac689d"}, + {file = "grpcio-1.62.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:f4aa94361bb5141a45ca9187464ae81a92a2a135ce2800b2203134f7a1a1d479"}, + {file = "grpcio-1.62.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82af3613a219512a28ee5c95578eb38d44dd03bca02fd918aa05603c41018051"}, + {file = "grpcio-1.62.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55ddaf53474e8caeb29eb03e3202f9d827ad3110475a21245f3c7712022882a9"}, + {file = "grpcio-1.62.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79b518c56dddeec79e5500a53d8a4db90da995dfe1738c3ac57fe46348be049"}, + {file = "grpcio-1.62.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5eb4844e5e60bf2c446ef38c5b40d7752c6effdee882f716eb57ae87255d20a"}, + {file = "grpcio-1.62.2-cp37-cp37m-win_amd64.whl", hash = "sha256:aaae70364a2d1fb238afd6cc9fcb10442b66e397fd559d3f0968d28cc3ac929c"}, + {file = "grpcio-1.62.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:1bcfe5070e4406f489e39325b76caeadab28c32bf9252d3ae960c79935a4cc36"}, + {file = "grpcio-1.62.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:da6a7b6b938c15fa0f0568e482efaae9c3af31963eec2da4ff13a6d8ec2888e4"}, + {file = "grpcio-1.62.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:41955b641c34db7d84db8d306937b72bc4968eef1c401bea73081a8d6c3d8033"}, + {file = "grpcio-1.62.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c772f225483905f675cb36a025969eef9712f4698364ecd3a63093760deea1bc"}, + {file = "grpcio-1.62.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07ce1f775d37ca18c7a141300e5b71539690efa1f51fe17f812ca85b5e73262f"}, + {file = "grpcio-1.62.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:26f415f40f4a93579fd648f48dca1c13dfacdfd0290f4a30f9b9aeb745026811"}, + {file = "grpcio-1.62.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:db707e3685ff16fc1eccad68527d072ac8bdd2e390f6daa97bc394ea7de4acea"}, + {file = "grpcio-1.62.2-cp38-cp38-win32.whl", hash = "sha256:589ea8e75de5fd6df387de53af6c9189c5231e212b9aa306b6b0d4f07520fbb9"}, + {file = "grpcio-1.62.2-cp38-cp38-win_amd64.whl", hash = "sha256:3c3ed41f4d7a3aabf0f01ecc70d6b5d00ce1800d4af652a549de3f7cf35c4abd"}, + {file = "grpcio-1.62.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:162ccf61499c893831b8437120600290a99c0bc1ce7b51f2c8d21ec87ff6af8b"}, + {file = "grpcio-1.62.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:f27246d7da7d7e3bd8612f63785a7b0c39a244cf14b8dd9dd2f2fab939f2d7f1"}, + {file = "grpcio-1.62.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:2507006c8a478f19e99b6fe36a2464696b89d40d88f34e4b709abe57e1337467"}, + {file = "grpcio-1.62.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a90ac47a8ce934e2c8d71e317d2f9e7e6aaceb2d199de940ce2c2eb611b8c0f4"}, + {file = "grpcio-1.62.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99701979bcaaa7de8d5f60476487c5df8f27483624f1f7e300ff4669ee44d1f2"}, + {file = "grpcio-1.62.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:af7dc3f7a44f10863b1b0ecab4078f0a00f561aae1edbd01fd03ad4dcf61c9e9"}, + {file = "grpcio-1.62.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fa63245271920786f4cb44dcada4983a3516be8f470924528cf658731864c14b"}, + {file = "grpcio-1.62.2-cp39-cp39-win32.whl", hash = "sha256:c6ad9c39704256ed91a1cffc1379d63f7d0278d6a0bad06b0330f5d30291e3a3"}, + {file = "grpcio-1.62.2-cp39-cp39-win_amd64.whl", hash = "sha256:16da954692fd61aa4941fbeda405a756cd96b97b5d95ca58a92547bba2c1624f"}, + {file = "grpcio-1.62.2.tar.gz", hash = "sha256:c77618071d96b7a8be2c10701a98537823b9c65ba256c0b9067e0594cdbd954d"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.62.1)"] +protobuf = ["grpcio-tools (>=1.62.2)"] [[package]] name = "grpcio-status" -version = "1.62.1" +version = "1.62.2" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.62.1.tar.gz", hash = "sha256:3431c8abbab0054912c41df5c72f03ddf3b7a67be8a287bb3c18a3456f96ff77"}, - {file = "grpcio_status-1.62.1-py3-none-any.whl", hash = "sha256:af0c3ab85da31669f21749e8d53d669c061ebc6ce5637be49a46edcb7aa8ab17"}, + {file = "grpcio-status-1.62.2.tar.gz", hash = "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a"}, + {file = "grpcio_status-1.62.2-py3-none-any.whl", hash = "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.62.1" +grpcio = ">=1.62.2" protobuf = ">=4.21.6" [[package]] @@ -1953,40 +1953,40 @@ dev = ["build (>=0.10.0)", "click (>=7.0,<8.2)", "hypothesis (>=5.8,<7)", "pdoc [[package]] name = "mitmproxy-macos" -version = "0.5.1" +version = "0.5.2" description = "" optional = false python-versions = ">=3.10" files = [ - {file = "mitmproxy_macos-0.5.1-py3-none-any.whl", hash = "sha256:3fb4fc9930b33101298675aeba6645dee71be17620c8cb07c810ba8bed6c2a42"}, + {file = "mitmproxy_macos-0.5.2-py3-none-any.whl", hash = "sha256:4aeee54ea4ecf7320b248292ef6dbc668ab14478efbdbf1234ae5ca120a13e63"}, ] [[package]] name = "mitmproxy-rs" -version = "0.5.1" +version = "0.5.2" description = "" optional = false python-versions = ">=3.10" files = [ - {file = "mitmproxy_rs-0.5.1-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5bfc3cf4a1f1dd09ee97ca8d9f2220ffeea29d5e9a0aa5a591deacf5612763c5"}, - {file = "mitmproxy_rs-0.5.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee18c0398dc439e9fe9d7dca66f1c2f868a6e0c2c444781c0b8964c794d1054f"}, - {file = "mitmproxy_rs-0.5.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2350fa71d0db814423eac65569be70d1788e8f4b8816cd56fc99be12a3498096"}, - {file = "mitmproxy_rs-0.5.1-cp310-abi3-win_amd64.whl", hash = "sha256:9e814163b5174c7ce65ef0c975f6ebf031ef1f3d4a0d8969644ec314108f91ab"}, - {file = "mitmproxy_rs-0.5.1.tar.gz", hash = "sha256:d8fc5dfde7bee019ebd0b29b28f178236949f3b4f229b9219929f15e2386d671"}, + {file = "mitmproxy_rs-0.5.2-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:c9e0c7136579adb5f23b3d12c40b392122276133e5cd1b2319ad0e01d1ec8ec0"}, + {file = "mitmproxy_rs-0.5.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45ca572479f32787de94b574dbedec042ab1d34d727d3597812fbdbd2f41922e"}, + {file = "mitmproxy_rs-0.5.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b4624e6b302d67fa94e50858a14a171708437e73146e3372ed042e01a09ca85"}, + {file = "mitmproxy_rs-0.5.2-cp310-abi3-win_amd64.whl", hash = "sha256:5e9f07b86b8a0f6a2c3c86c1fe902070e65868a0cf4d668ca7d1e2a802fe6e3f"}, + {file = "mitmproxy_rs-0.5.2.tar.gz", hash = "sha256:7583bea1ff5ea8e96c5cf12127e1698c52725f1dfdac6802891a4675b7287ba5"}, ] [package.dependencies] -mitmproxy_macos = {version = "0.5.1", markers = "sys_platform == \"darwin\""} -mitmproxy_windows = {version = "0.5.1", markers = "os_name == \"nt\""} +mitmproxy_macos = {version = "0.5.2", markers = "sys_platform == \"darwin\""} +mitmproxy_windows = {version = "0.5.2", markers = "os_name == \"nt\""} [[package]] name = "mitmproxy-windows" -version = "0.5.1" +version = "0.5.2" description = "" optional = false python-versions = ">=3.10" files = [ - {file = "mitmproxy_windows-0.5.1-py3-none-any.whl", hash = "sha256:08c2e71f9b7ff6aa094943627646f9afe048ec20ad892b701d1aba7de145e15a"}, + {file = "mitmproxy_windows-0.5.2-py3-none-any.whl", hash = "sha256:e7834cd4825a55d703b4aed34d2d7f85a2749ccb86396e328339070e528a3561"}, ] [[package]] @@ -2155,38 +2155,38 @@ files = [ [[package]] name = "mypy" -version = "1.9.0" +version = "1.10.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"}, - {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"}, - {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"}, - {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"}, - {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"}, - {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"}, - {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"}, - {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"}, - {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"}, - {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"}, - {file = "mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd"}, - {file = "mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6"}, - {file = "mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185"}, - {file = "mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913"}, - {file = "mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6"}, - {file = "mypy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b"}, - {file = "mypy-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2"}, - {file = "mypy-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e"}, - {file = "mypy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04"}, - {file = "mypy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89"}, - {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"}, - {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"}, - {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"}, - {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"}, - {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"}, - {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"}, - {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, + {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, + {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, + {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, + {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, + {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, + {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, + {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, + {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, + {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, + {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, + {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, + {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, + {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, + {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, + {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, + {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, + {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, ] [package.dependencies] @@ -2318,6 +2318,7 @@ files = [ {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, @@ -2443,28 +2444,29 @@ scramp = ">=1.4.4" [[package]] name = "platformdirs" -version = "4.2.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "4.2.1" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, - {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, + {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, + {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, ] [package.extras] docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] [[package]] name = "pluggy" -version = "1.4.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, - {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -2521,51 +2523,51 @@ files = [ [[package]] name = "pyarrow" -version = "15.0.2" +version = "16.0.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.8" files = [ - {file = "pyarrow-15.0.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:88b340f0a1d05b5ccc3d2d986279045655b1fe8e41aba6ca44ea28da0d1455d8"}, - {file = "pyarrow-15.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eaa8f96cecf32da508e6c7f69bb8401f03745c050c1dd42ec2596f2e98deecac"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23c6753ed4f6adb8461e7c383e418391b8d8453c5d67e17f416c3a5d5709afbd"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f639c059035011db8c0497e541a8a45d98a58dbe34dc8fadd0ef128f2cee46e5"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:290e36a59a0993e9a5224ed2fb3e53375770f07379a0ea03ee2fce2e6d30b423"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:06c2bb2a98bc792f040bef31ad3e9be6a63d0cb39189227c08a7d955db96816e"}, - {file = "pyarrow-15.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:f7a197f3670606a960ddc12adbe8075cea5f707ad7bf0dffa09637fdbb89f76c"}, - {file = "pyarrow-15.0.2-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:5f8bc839ea36b1f99984c78e06e7a06054693dc2af8920f6fb416b5bca9944e4"}, - {file = "pyarrow-15.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5e81dfb4e519baa6b4c80410421528c214427e77ca0ea9461eb4097c328fa33"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a4f240852b302a7af4646c8bfe9950c4691a419847001178662a98915fd7ee7"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e7d9cfb5a1e648e172428c7a42b744610956f3b70f524aa3a6c02a448ba853e"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2d4f905209de70c0eb5b2de6763104d5a9a37430f137678edfb9a675bac9cd98"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90adb99e8ce5f36fbecbbc422e7dcbcbed07d985eed6062e459e23f9e71fd197"}, - {file = "pyarrow-15.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:b116e7fd7889294cbd24eb90cd9bdd3850be3738d61297855a71ac3b8124ee38"}, - {file = "pyarrow-15.0.2-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:25335e6f1f07fdaa026a61c758ee7d19ce824a866b27bba744348fa73bb5a440"}, - {file = "pyarrow-15.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:90f19e976d9c3d8e73c80be84ddbe2f830b6304e4c576349d9360e335cd627fc"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a22366249bf5fd40ddacc4f03cd3160f2d7c247692945afb1899bab8a140ddfb"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2a335198f886b07e4b5ea16d08ee06557e07db54a8400cc0d03c7f6a22f785f"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e6d459c0c22f0b9c810a3917a1de3ee704b021a5fb8b3bacf968eece6df098f"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:033b7cad32198754d93465dcfb71d0ba7cb7cd5c9afd7052cab7214676eec38b"}, - {file = "pyarrow-15.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:29850d050379d6e8b5a693098f4de7fd6a2bea4365bfd073d7c57c57b95041ee"}, - {file = "pyarrow-15.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7167107d7fb6dcadb375b4b691b7e316f4368f39f6f45405a05535d7ad5e5058"}, - {file = "pyarrow-15.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e85241b44cc3d365ef950432a1b3bd44ac54626f37b2e3a0cc89c20e45dfd8bf"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:248723e4ed3255fcd73edcecc209744d58a9ca852e4cf3d2577811b6d4b59818"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ff3bdfe6f1b81ca5b73b70a8d482d37a766433823e0c21e22d1d7dde76ca33f"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f3d77463dee7e9f284ef42d341689b459a63ff2e75cee2b9302058d0d98fe142"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:8c1faf2482fb89766e79745670cbca04e7018497d85be9242d5350cba21357e1"}, - {file = "pyarrow-15.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:28f3016958a8e45a1069303a4a4f6a7d4910643fc08adb1e2e4a7ff056272ad3"}, - {file = "pyarrow-15.0.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:89722cb64286ab3d4daf168386f6968c126057b8c7ec3ef96302e81d8cdb8ae4"}, - {file = "pyarrow-15.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd0ba387705044b3ac77b1b317165c0498299b08261d8122c96051024f953cd5"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad2459bf1f22b6a5cdcc27ebfd99307d5526b62d217b984b9f5c974651398832"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58922e4bfece8b02abf7159f1f53a8f4d9f8e08f2d988109126c17c3bb261f22"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:adccc81d3dc0478ea0b498807b39a8d41628fa9210729b2f718b78cb997c7c91"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:8bd2baa5fe531571847983f36a30ddbf65261ef23e496862ece83bdceb70420d"}, - {file = "pyarrow-15.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6669799a1d4ca9da9c7e06ef48368320f5856f36f9a4dd31a11839dda3f6cc8c"}, - {file = "pyarrow-15.0.2.tar.gz", hash = "sha256:9c9bc803cb3b7bfacc1e96ffbfd923601065d9d3f911179d81e72d99fd74a3d9"}, -] - -[package.dependencies] -numpy = ">=1.16.6,<2" + {file = "pyarrow-16.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:22a1fdb1254e5095d629e29cd1ea98ed04b4bbfd8e42cc670a6b639ccc208b60"}, + {file = "pyarrow-16.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:574a00260a4ed9d118a14770edbd440b848fcae5a3024128be9d0274dbcaf858"}, + {file = "pyarrow-16.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0815d0ddb733b8c1b53a05827a91f1b8bde6240f3b20bf9ba5d650eb9b89cdf"}, + {file = "pyarrow-16.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df0080339387b5d30de31e0a149c0c11a827a10c82f0c67d9afae3981d1aabb7"}, + {file = "pyarrow-16.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:edf38cce0bf0dcf726e074159c60516447e4474904c0033f018c1f33d7dac6c5"}, + {file = "pyarrow-16.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:91d28f9a40f1264eab2af7905a4d95320ac2f287891e9c8b0035f264fe3c3a4b"}, + {file = "pyarrow-16.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:99af421ee451a78884d7faea23816c429e263bd3618b22d38e7992c9ce2a7ad9"}, + {file = "pyarrow-16.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:d22d0941e6c7bafddf5f4c0662e46f2075850f1c044bf1a03150dd9e189427ce"}, + {file = "pyarrow-16.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:266ddb7e823f03733c15adc8b5078db2df6980f9aa93d6bb57ece615df4e0ba7"}, + {file = "pyarrow-16.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cc23090224b6594f5a92d26ad47465af47c1d9c079dd4a0061ae39551889efe"}, + {file = "pyarrow-16.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56850a0afe9ef37249d5387355449c0f94d12ff7994af88f16803a26d38f2016"}, + {file = "pyarrow-16.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:705db70d3e2293c2f6f8e84874b5b775f690465798f66e94bb2c07bab0a6bb55"}, + {file = "pyarrow-16.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5448564754c154997bc09e95a44b81b9e31ae918a86c0fcb35c4aa4922756f55"}, + {file = "pyarrow-16.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:729f7b262aa620c9df8b9967db96c1575e4cfc8c25d078a06968e527b8d6ec05"}, + {file = "pyarrow-16.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:fb8065dbc0d051bf2ae2453af0484d99a43135cadabacf0af588a3be81fbbb9b"}, + {file = "pyarrow-16.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:20ce707d9aa390593ea93218b19d0eadab56390311cb87aad32c9a869b0e958c"}, + {file = "pyarrow-16.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5823275c8addbbb50cd4e6a6839952682a33255b447277e37a6f518d6972f4e1"}, + {file = "pyarrow-16.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ab8b9050752b16a8b53fcd9853bf07d8daf19093533e990085168f40c64d978"}, + {file = "pyarrow-16.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:42e56557bc7c5c10d3e42c3b32f6cff649a29d637e8f4e8b311d334cc4326730"}, + {file = "pyarrow-16.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2a7abdee4a4a7cfa239e2e8d721224c4b34ffe69a0ca7981354fe03c1328789b"}, + {file = "pyarrow-16.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ef2f309b68396bcc5a354106741d333494d6a0d3e1951271849787109f0229a6"}, + {file = "pyarrow-16.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:ed66e5217b4526fa3585b5e39b0b82f501b88a10d36bd0d2a4d8aa7b5a48e2df"}, + {file = "pyarrow-16.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc8814310486f2a73c661ba8354540f17eef51e1b6dd090b93e3419d3a097b3a"}, + {file = "pyarrow-16.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c2f5e239db7ed43e0ad2baf46a6465f89c824cc703f38ef0fde927d8e0955f7"}, + {file = "pyarrow-16.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f293e92d1db251447cb028ae12f7bc47526e4649c3a9924c8376cab4ad6b98bd"}, + {file = "pyarrow-16.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:dd9334a07b6dc21afe0857aa31842365a62eca664e415a3f9536e3a8bb832c07"}, + {file = "pyarrow-16.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d91073d1e2fef2c121154680e2ba7e35ecf8d4969cc0af1fa6f14a8675858159"}, + {file = "pyarrow-16.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:71d52561cd7aefd22cf52538f262850b0cc9e4ec50af2aaa601da3a16ef48877"}, + {file = "pyarrow-16.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b93c9a50b965ee0bf4fef65e53b758a7e8dcc0c2d86cebcc037aaaf1b306ecc0"}, + {file = "pyarrow-16.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d831690844706e374c455fba2fb8cfcb7b797bfe53ceda4b54334316e1ac4fa4"}, + {file = "pyarrow-16.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35692ce8ad0b8c666aa60f83950957096d92f2a9d8d7deda93fb835e6053307e"}, + {file = "pyarrow-16.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dd3151d098e56f16a8389c1247137f9e4c22720b01c6f3aa6dec29a99b74d80"}, + {file = "pyarrow-16.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:bd40467bdb3cbaf2044ed7a6f7f251c8f941c8b31275aaaf88e746c4f3ca4a7a"}, + {file = "pyarrow-16.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:00a1dcb22ad4ceb8af87f7bd30cc3354788776c417f493089e0a0af981bc8d80"}, + {file = "pyarrow-16.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fda9a7cebd1b1d46c97b511f60f73a5b766a6de4c5236f144f41a5d5afec1f35"}, + {file = "pyarrow-16.0.0.tar.gz", hash = "sha256:59bb1f1edbbf4114c72415f039f1359f1a57d166a331c3229788ccbfbb31689a"}, +] + +[package.dependencies] +numpy = ">=1.16.6" [[package]] name = "pyasn1" @@ -2950,6 +2952,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -3560,17 +3563,17 @@ files = [ [[package]] name = "websocket-client" -version = "1.7.0" +version = "1.8.0" description = "WebSocket client for Python with low level API options" optional = false python-versions = ">=3.8" files = [ - {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, - {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, + {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"}, + {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"}, ] [package.extras] -docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] +docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] @@ -3783,4 +3786,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10,<3.12" -content-hash = "f1f2107fe0456326deb5c0c0affdebeaf78049bd7bd931e9ff4b7665dccbd759" +content-hash = "c23821b61ee592ec43f11c22e1e3b5eb597eefe17265fc546e4fabffed697a05" diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml index 4317e5dd7fbd..b399319e88a0 100644 --- a/airbyte-ci/connectors/live-tests/pyproject.toml +++ b/airbyte-ci/connectors/live-tests/pyproject.toml @@ -57,8 +57,11 @@ select = ["I", "F"] known-first-party = ["connection-retriever"] [tool.poe.tasks] -format = "ruff format src" test = "pytest tests" -lint = "ruff check src" type_check = "mypy src --disallow-untyped-defs" -pre-push = ["format", "lint", "test", "type_check"] +pre-push = [] + +[tool.airbyte_ci] +optional_poetry_groups = ["dev"] +poe_tasks = [] +required_environment_variables = ["DOCKER_HUB_USERNAME", "DOCKER_HUB_PASSWORD"] diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/cli.py b/airbyte-ci/connectors/live-tests/src/live_tests/cli.py index 5c7e22e56dad..f23da702fe0b 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/cli.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/cli.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import asyncclick as click from live_tests.debug.cli import debug_cmd diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/__init__.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/__init__.py index 177ff35cf687..62501987fb84 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/__init__.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from .base_backend import BaseBackend from .duckdb_backend import DuckDbBackend diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py index f009b8272275..50a0209655cb 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py @@ -1,7 +1,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from abc import ABC, abstractmethod -from typing import Iterable +from collections.abc import Iterable from airbyte_protocol.models import AirbyteMessage # type: ignore diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py index 41f7518d3ae5..cd6d61ee5d6c 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - +from __future__ import annotations import logging import re +from collections.abc import Iterable from pathlib import Path -from typing import Iterable, Optional +from typing import Optional import duckdb from airbyte_protocol.models import AirbyteMessage # type: ignore diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py index 72620d3de502..a4d0b57c910a 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py @@ -1,9 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import json import logging +from collections.abc import Iterable from pathlib import Path -from typing import Any, Dict, Iterable, TextIO, Tuple +from typing import Any, TextIO from airbyte_protocol.models import AirbyteMessage # type: ignore from airbyte_protocol.models import Type as AirbyteMessageType @@ -13,7 +15,7 @@ class FileDescriptorLRUCache(LRUCache): - def popitem(self) -> Tuple[Any, Any]: + def popitem(self) -> tuple[Any, Any]: filepath, fd = LRUCache.popitem(self) fd.close() # type: ignore # Close the file descriptor when it's evicted from the cache return filepath, fd @@ -34,8 +36,8 @@ def __init__(self, output_directory: Path): self._output_directory = output_directory self.record_per_stream_directory = self._output_directory / "records_per_stream" self.record_per_stream_directory.mkdir(exist_ok=True, parents=True) - self.record_per_stream_paths: Dict[str, Path] = {} - self.record_per_stream_paths_data_only: Dict[str, Path] = {} + self.record_per_stream_paths: dict[str, Path] = {} + self.record_per_stream_paths_data_only: dict[str, Path] = {} @property def jsonl_specs_path(self) -> Path: @@ -101,14 +103,14 @@ def _open_file(path: Path) -> TextIO: if not isinstance(_message, AirbyteMessage): continue filepaths, messages = self._get_filepaths_and_messages(_message) - for filepath, message in zip(filepaths, messages): + for filepath, message in zip(filepaths, messages, strict=False): _open_file(self._output_directory / filepath).write(f"{message}\n") logging.info("Finished writing airbyte messages to disk") finally: for f in self.CACHE.values(): f.close() - def _get_filepaths_and_messages(self, message: AirbyteMessage) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: + def _get_filepaths_and_messages(self, message: AirbyteMessage) -> tuple[tuple[str, ...], tuple[str, ...]]: if message.type == AirbyteMessageType.CATALOG: return (self.RELATIVE_CATALOGS_PATH,), (message.catalog.json(),) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py index e9e263696e58..1d8e75253e62 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py @@ -1,10 +1,11 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations import json import logging import os from pathlib import Path -from typing import Dict, Optional, Set +from typing import Optional import rich from connection_retriever import ConnectionObject, retrieve_objects # type: ignore @@ -16,7 +17,7 @@ console = rich.get_console() -def parse_config(config: Dict | str | None) -> Optional[SecretDict]: +def parse_config(config: dict | str | None) -> Optional[SecretDict]: if not config: return None if isinstance(config, str): @@ -25,7 +26,7 @@ def parse_config(config: Dict | str | None) -> Optional[SecretDict]: return SecretDict(config) -def parse_catalog(catalog: Dict | str | None) -> Optional[AirbyteCatalog]: +def parse_catalog(catalog: dict | str | None) -> Optional[AirbyteCatalog]: if not catalog: return None if isinstance(catalog, str): @@ -35,7 +36,7 @@ def parse_catalog(catalog: Dict | str | None) -> Optional[AirbyteCatalog]: def parse_configured_catalog( - configured_catalog: Dict | str | None, selected_streams: Set[str] | None = None + configured_catalog: dict | str | None, selected_streams: set[str] | None = None ) -> Optional[ConfiguredAirbyteCatalog]: if not configured_catalog: return None @@ -48,7 +49,7 @@ def parse_configured_catalog( return catalog -def parse_state(state: Dict | str | None) -> Optional[Dict]: +def parse_state(state: dict | str | None) -> Optional[dict]: if not state: return None if isinstance(state, str): @@ -61,11 +62,11 @@ def get_connector_config_from_path(config_path: Path) -> Optional[SecretDict]: return parse_config(config_path.read_text()) -def get_state_from_path(state_path: Path) -> Optional[Dict]: +def get_state_from_path(state_path: Path) -> Optional[dict]: return parse_state(state_path.read_text()) -def get_configured_catalog_from_path(path: Path, selected_streams: Optional[Set[str]] = None) -> Optional[ConfiguredAirbyteCatalog]: +def get_configured_catalog_from_path(path: Path, selected_streams: Optional[set[str]] = None) -> Optional[ConfiguredAirbyteCatalog]: return parse_configured_catalog(path.read_text(), selected_streams) @@ -83,7 +84,7 @@ def get_configured_catalog_from_path(path: Path, selected_streams: Optional[Set[ def get_connection_objects( - requested_objects: Set[ConnectionObject], + requested_objects: set[ConnectionObject], connection_id: Optional[str], custom_config_path: Optional[Path], custom_configured_catalog_path: Optional[Path], @@ -92,7 +93,7 @@ def get_connection_objects( fail_if_missing_objects: bool = True, connector_image: Optional[str] = None, auto_select_connection: bool = False, - selected_streams: Optional[Set[str]] = None, + selected_streams: Optional[set[str]] = None, ) -> ConnectionObjects: """This function retrieves the connection objects values. It checks that the required objects are available and raises a UsageError if they are not. diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py index b7ba8ef8fd08..3e5838dd7d8e 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py @@ -1,13 +1,14 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations import datetime import json import logging import uuid from pathlib import Path -from typing import List, Optional +from typing import Optional import anyio import asyncer @@ -39,7 +40,7 @@ def __init__( self.actor_id = execution_inputs.actor_id self.environment_variables = execution_inputs.environment_variables if execution_inputs.environment_variables else {} - self.full_command: List[str] = self._get_full_command(execution_inputs.command) + self.full_command: list[str] = self._get_full_command(execution_inputs.command) self.completion_event = anyio.Event() self.http_proxy = http_proxy self.logger = logging.getLogger(f"{self.connector_under_test.name}-{self.connector_under_test.version}") @@ -57,7 +58,7 @@ def stdout_file_path(self) -> Path: def stderr_file_path(self) -> Path: return (self.output_dir / "stderr.log").resolve() - def _get_full_command(self, command: Command) -> List[str]: + def _get_full_command(self, command: Command) -> list[str]: if command is Command.SPEC: return ["spec"] elif command is Command.CHECK: @@ -184,7 +185,7 @@ async def _log_progress(self) -> None: def format_duration(time_delta: datetime.timedelta) -> str: total_seconds = time_delta.total_seconds() if total_seconds < 60: - return "{:.2f}s".format(total_seconds) + return f"{total_seconds:.2f}s" minutes = int(total_seconds // 60) seconds = int(total_seconds % 60) - return "{:02d}mn{:02d}s".format(minutes, seconds) + return f"{minutes:02d}mn{seconds:02d}s" diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/errors.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/errors.py index 402429cfb2d5..cb13b4ab629e 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/errors.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/errors.py @@ -1,4 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations class ExportError(Exception): diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/mitm_addons.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/mitm_addons.py index d650c843f217..5bcfce2dafd5 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/mitm_addons.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/mitm_addons.py @@ -1,4 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations from urllib.parse import parse_qs, urlencode, urlparse diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py index f6b6acefff95..6b0a6b406a28 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py @@ -5,17 +5,18 @@ import logging import tempfile from collections import defaultdict +from collections.abc import Iterable, Iterator, MutableMapping from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, List, MutableMapping, Optional, Type +from typing import Any, Optional import _collections_abc import dagger import requests - -# type: ignore -from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, ConfiguredAirbyteCatalog # type: ignore +from airbyte_protocol.models import AirbyteCatalog # type: ignore +from airbyte_protocol.models import AirbyteMessage # type: ignore +from airbyte_protocol.models import ConfiguredAirbyteCatalog # type: ignore from airbyte_protocol.models import Type as AirbyteMessageType from genson import SchemaBuilder # type: ignore from live_tests.commons.backends import DuckDbBackend, FileBackend @@ -174,7 +175,7 @@ def actor_type(self) -> ActorType: @classmethod async def from_image_name( - cls: Type[ConnectorUnderTest], + cls: type[ConnectorUnderTest], dagger_client: dagger.Client, image_name: str, target_or_control: TargetOrControl, @@ -191,8 +192,8 @@ class ExecutionInputs: command: Command config: Optional[SecretDict] = None configured_catalog: Optional[ConfiguredAirbyteCatalog] = None - state: Optional[Dict] = None - environment_variables: Optional[Dict] = None + state: Optional[dict] = None + environment_variables: Optional[dict] = None duckdb_path: Optional[Path] = None def raise_if_missing_attr_for_command(self, attribute: str) -> None: @@ -232,8 +233,8 @@ class ExecutionResult: success: bool executed_container: Optional[dagger.Container] http_dump: Optional[dagger.File] = None - http_flows: List[http.HTTPFlow] = field(default_factory=list) - stream_schemas: Optional[Dict[str, Any]] = None + http_flows: list[http.HTTPFlow] = field(default_factory=list) + stream_schemas: Optional[dict[str, Any]] = None backend: Optional[FileBackend] = None HTTP_DUMP_FILE_NAME = "http_dump.mitm" @@ -253,7 +254,7 @@ def duckdb_schema(self) -> Iterable[str]: @classmethod async def load( - cls: Type[ExecutionResult], + cls: type[ExecutionResult], connector_under_test: ConnectorUnderTest, actor_id: str, command: Command, @@ -286,7 +287,7 @@ async def load_http_flows(self) -> None: def parse_airbyte_messages_from_command_output( self, command_output_path: Path, log_validation_errors: bool = False ) -> Iterable[AirbyteMessage]: - with open(command_output_path, "r") as command_output: + with open(command_output_path) as command_output: for line in command_output: try: yield AirbyteMessage.parse_raw(line) @@ -302,9 +303,9 @@ def get_records(self) -> Iterable[AirbyteMessage]: if message.type is AirbyteMessageType.RECORD: yield message - def generate_stream_schemas(self) -> Dict[str, Any]: + def generate_stream_schemas(self) -> dict[str, Any]: self.logger.info("Generating stream schemas") - stream_builders: Dict[str, SchemaBuilder] = {} + stream_builders: dict[str, SchemaBuilder] = {} for record in self.get_records(): stream = record.record.stream if stream not in stream_builders: @@ -328,8 +329,8 @@ def get_records_per_stream(self, stream: str) -> Iterator[AirbyteMessage]: if message.type is AirbyteMessageType.RECORD: yield message - def get_message_count_per_type(self) -> Dict[AirbyteMessageType, int]: - message_count: Dict[AirbyteMessageType, int] = defaultdict(int) + def get_message_count_per_type(self) -> dict[AirbyteMessageType, int]: + message_count: dict[AirbyteMessageType, int] = defaultdict(int) for message in self.airbyte_messages: message_count[message.type] += 1 return message_count @@ -376,7 +377,7 @@ async def save_artifacts(self, output_dir: Path, duckdb_path: Optional[Path] = N self.save_stream_schemas(output_dir) self.logger.info("All artifacts saved to disk") - def get_updated_configuration(self, control_message_path: Path) -> Optional[Dict[str, Any]]: + def get_updated_configuration(self, control_message_path: Path) -> Optional[dict[str, Any]]: """Iterate through the control messages to find CONNECTOR_CONFIG message and return the last updated configuration.""" if not control_message_path.exists(): return None @@ -403,7 +404,7 @@ def update_configuration(self) -> None: payload = { "configuration": { **updated_configuration, - **{f"{self.connector_under_test.actor_type.value}Type": self.connector_under_test.name_without_type_prefix}, + f"{self.connector_under_test.actor_type.value}Type": self.connector_under_test.name_without_type_prefix, } } headers = { @@ -427,7 +428,7 @@ class ConnectionObjects: destination_config: Optional[SecretDict] configured_catalog: Optional[ConfiguredAirbyteCatalog] catalog: Optional[AirbyteCatalog] - state: Optional[Dict] + state: Optional[dict] workspace_id: Optional[str] source_id: Optional[str] destination_id: Optional[str] diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py index a50f7f4c3780..4627b1024ee8 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py @@ -1,4 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations import logging import uuid diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/secret_access.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/secret_access.py index 1545cc8ce25b..260953e1971c 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/secret_access.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/secret_access.py @@ -1,4 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations import logging @@ -26,7 +27,7 @@ def get_secret_value(secret_manager_client: secretmanager.SecretManagerServiceCl response = secret_manager_client.access_secret_version(name=enabled_version.name) return response.payload.data.decode("UTF-8") except PermissionDenied as e: - logging.error( + logging.exception( f"Permission denied while trying to access secret {secret_id}. Please write to #dev-extensibility in Airbyte Slack for help.", exc_info=e, ) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/segment_tracking.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/segment_tracking.py index 7427cf73eca9..a57a2aa542bb 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/segment_tracking.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/segment_tracking.py @@ -1,11 +1,12 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations import logging import os from importlib.metadata import version -from typing import Any, Dict +from typing import Any, Optional -import segment.analytics as analytics # type: ignore +from segment import analytics # type: ignore ENABLE_TRACKING = os.getenv("REGRESSION_TEST_DISABLE_TRACKING") is None DEBUG_SEGMENT = os.getenv("DEBUG_SEGMENT") is not None @@ -25,10 +26,14 @@ def on_error(error: Exception, items: Any) -> None: def track_usage( - user_id: str, - pytest_options: Dict[str, Any], + user_id: Optional[str], + pytest_options: dict[str, Any], ) -> None: - analytics.identify(user_id) + if user_id: + analytics.identify(user_id) + else: + user_id = "airbyte-ci" + # It contains default pytest option and the custom one passed by the user analytics.track( user_id, diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py index 9e4244c4b20a..a30342f926cf 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/utils.py @@ -1,11 +1,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import logging import os import re import shutil from pathlib import Path -from typing import List, Optional +from typing import Optional import dagger import docker # type: ignore @@ -96,10 +97,9 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ # If a container_id.txt file is available, we'll use it to load the connector container # We use a txt file as container ids can be too long to be passed as env vars # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation - connector_container_id_path = Path("/tmp/container_id.txt") - if connector_container_id_path.exists(): - # If the CONNECTOR_CONTAINER_ID env var is set, we'll use it to load the connector container - return await get_container_from_id(dagger_client, connector_container_id_path.read_text()) + container_id_path = Path("/tmp/container_id.txt") + if container_id_path.exists(): + return await get_container_from_id(dagger_client, container_id_path.read_text()) # If the CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH env var is set, we'll use it to import the connector image from the tarball if connector_image_tarball_path := os.environ.get("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"): @@ -114,7 +114,7 @@ async def get_connector_container(dagger_client: dagger.Client, image_name_with_ return await get_container_from_dockerhub_image(dagger_client, image_name_with_tag) -def sh_dash_c(lines: List[str]) -> List[str]: +def sh_dash_c(lines: list[str]) -> list[str]: """Wrap sequence of commands in shell for safe usage of dagger Container's with_exec method.""" return ["sh", "-c", " && ".join(["set -o xtrace"] + lines)] @@ -125,7 +125,7 @@ def clean_up_artifacts(directory: Path, logger: logging.Logger) -> None: logger.info(f"🧹 Test artifacts cleaned up from {directory}") -def get_http_flows_from_mitm_dump(mitm_dump_path: Path) -> List[http.HTTPFlow]: +def get_http_flows_from_mitm_dump(mitm_dump_path: Path) -> list[http.HTTPFlow]: """Get http flows from a mitmproxy dump file. Args: diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py b/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py index 3ffc8dc5255f..d3a001370f73 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/debug/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import os import sys diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py b/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py index 37556ec7bf54..47f24e3db645 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/debug/cli.py @@ -1,9 +1,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import logging import textwrap from pathlib import Path -from typing import List, Optional +from typing import Optional import asyncclick as click import dagger @@ -65,7 +66,7 @@ async def debug_cmd( config_path: Optional[Path], catalog_path: Optional[Path], state_path: Optional[Path], - connector_images: List[str], + connector_images: list[str], ) -> None: if connection_id: retrieval_reason = click.prompt("👮♂️ Please provide a reason for accessing the connection objects. This will be logged") diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py b/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py index 855a6c8c5d10..192881be5915 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/debug/consts.py @@ -1,4 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations from pathlib import Path diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py index 2c45d88c0bdf..3fc758b63b52 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/conftest.py @@ -6,8 +6,9 @@ import textwrap import time import webbrowser +from collections.abc import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable from pathlib import Path -from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Callable, Dict, Generator, Iterable, List, Optional, Set +from typing import TYPE_CHECKING, Optional import dagger import pytest @@ -73,24 +74,46 @@ def pytest_addoption(parser: Parser) -> None: help="Automatically select the connection to run the tests on.", ) parser.addoption("--pr-url", help="The URL of the PR you are testing") - parser.addoption("--stream", help="The stream to run the tests on. (Can be used multiple times)", action="append") + parser.addoption( + "--stream", + help="The stream to run the tests on. (Can be used multiple times)", + action="append", + ) + # Required when running in CI + parser.addoption("--run-id", type=str) + parser.addoption( + "--should-read-with-state", + type=bool, + help="Whether to run the `read` command with state. \n" + "We recommend reading with state to properly test incremental sync. \n" + "But if the target version introduces a breaking change in the state, you might want to run without state. \n", + ) def pytest_configure(config: Config) -> None: user_email = get_user_email() + config.stash[stash_keys.RUN_IN_AIRBYTE_CI] = bool(os.getenv("RUN_IN_AIRBYTE_CI", False)) + config.stash[stash_keys.IS_PRODUCTION_CI] = bool(os.getenv("CI", False)) prompt_for_confirmation(user_email) - track_usage(user_email, vars(config.option)) + track_usage( + "production-ci" + if config.stash[stash_keys.IS_PRODUCTION_CI] + else "local-ci" + if config.stash[stash_keys.RUN_IN_AIRBYTE_CI] + else user_email, + vars(config.option), + ) config.stash[stash_keys.AIRBYTE_API_KEY] = get_airbyte_api_key() config.stash[stash_keys.USER] = user_email - start_timestamp = int(time.time()) - test_artifacts_directory = MAIN_OUTPUT_DIRECTORY / f"session_{start_timestamp}" + config.stash[stash_keys.SESSION_RUN_ID] = config.getoption("--run-id") or str(int(time.time())) + test_artifacts_directory = get_artifacts_directory(config) duckdb_path = test_artifacts_directory / "duckdb.db" config.stash[stash_keys.DUCKDB_PATH] = duckdb_path test_artifacts_directory.mkdir(parents=True, exist_ok=True) dagger_log_path = test_artifacts_directory / "dagger.log" config.stash[stash_keys.IS_PERMITTED_BOOL] = False report_path = test_artifacts_directory / "report.html" - config.stash[stash_keys.SESSION_START_TIMESTAMP] = start_timestamp + config.stash[stash_keys.TEST_ARTIFACT_DIRECTORY] = test_artifacts_directory dagger_log_path.touch() config.stash[stash_keys.DAGGER_LOG_PATH] = dagger_log_path @@ -103,8 +126,15 @@ def pytest_configure(config: Config) -> None: custom_state_path = config.getoption("--state-path") config.stash[stash_keys.SELECTED_STREAMS] = set(config.getoption("--stream") or []) - config.stash[stash_keys.SHOULD_READ_WITH_STATE] = prompt_for_read_with_or_without_state() + if config.stash[stash_keys.RUN_IN_AIRBYTE_CI]: + config.stash[stash_keys.SHOULD_READ_WITH_STATE] = bool(get_option_or_fail(config, "--should-read-with-state")) + elif _should_read_with_state := config.getoption("--should-read-with-state"): + config.stash[stash_keys.SHOULD_READ_WITH_STATE] = _should_read_with_state + else: + config.stash[stash_keys.SHOULD_READ_WITH_STATE] = prompt_for_read_with_or_without_state() + retrieval_reason = f"Running regression tests on connection for connector {config.stash[stash_keys.CONNECTOR_IMAGE]} on target versions ({config.stash[stash_keys.TARGET_VERSION]})." + try: config.stash[stash_keys.CONNECTION_OBJECTS] = get_connection_objects( { @@ -155,7 +185,12 @@ def pytest_configure(config: Config) -> None: webbrowser.open_new_tab(config.stash[stash_keys.REPORT].path.resolve().as_uri()) -def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item]) -> None: +def get_artifacts_directory(config: pytest.Config) -> Path: + run_id = config.stash[stash_keys.SESSION_RUN_ID] + return MAIN_OUTPUT_DIRECTORY / f"session_{run_id}" + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: for item in items: if config.stash[stash_keys.SHOULD_READ_WITH_STATE] and "without_state" in item.keywords: item.add_marker(pytest.mark.skip(reason="Test is marked with without_state marker")) @@ -176,18 +211,19 @@ def pytest_terminal_summary(terminalreporter: SugarTerminalReporter, exitstatus: f"All tests artifacts for this sessions should be available in {config.stash[stash_keys.TEST_ARTIFACT_DIRECTORY].resolve()}" ) - try: - Prompt.ask( - textwrap.dedent( - """ - Test artifacts will be destroyed after this prompt. - Press enter when you're done reading them. - 🚨 Do not copy them elsewhere on your disk!!! 🚨 - """ + if not config.stash[stash_keys.RUN_IN_AIRBYTE_CI]: + try: + Prompt.ask( + textwrap.dedent( + """ + Test artifacts will be destroyed after this prompt. + Press enter when you're done reading them. + 🚨 Do not copy them elsewhere on your disk!!! 🚨 + """ + ) ) - ) - finally: - clean_up_artifacts(MAIN_OUTPUT_DIRECTORY, LOGGER) + finally: + clean_up_artifacts(MAIN_OUTPUT_DIRECTORY, LOGGER) def pytest_keyboard_interrupt(excinfo: Exception) -> None: @@ -200,13 +236,7 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo) -> Gener outcome = yield report = outcome.get_result() # This is to add skipped or failed tests due to upstream fixture failures on setup - if report.outcome in ["failed", "skipped"]: - item.config.stash[stash_keys.REPORT].add_test_result( - report, - item.function.__doc__, # type: ignore - ) - - elif report.when == "call": + if report.outcome in ["failed", "skipped"] or report.when == "call": item.config.stash[stash_keys.REPORT].add_test_result( report, item.function.__doc__, # type: ignore @@ -267,11 +297,6 @@ def anyio_backend() -> str: return "asyncio" -@pytest.fixture(scope="session") -def session_start_timestamp(request: SubRequest) -> int: - return request.config.stash[stash_keys.SESSION_START_TIMESTAMP] - - @pytest.fixture(scope="session") def test_artifacts_directory(request: SubRequest) -> Path: return request.config.stash[stash_keys.TEST_ARTIFACT_DIRECTORY] @@ -318,12 +343,12 @@ def actor_id(connection_objects: ConnectionObjects, control_connector: Connector @pytest.fixture(scope="session") -def selected_streams(request: SubRequest) -> Set[str]: +def selected_streams(request: SubRequest) -> set[str]: return request.config.stash[stash_keys.SELECTED_STREAMS] @pytest.fixture(scope="session") -def configured_catalog(connection_objects: ConnectionObjects, selected_streams: Optional[Set[str]]) -> ConfiguredAirbyteCatalog: +def configured_catalog(connection_objects: ConnectionObjects, selected_streams: Optional[set[str]]) -> ConfiguredAirbyteCatalog: if not connection_objects.configured_catalog: pytest.skip("Catalog is not provided. The catalog fixture can't be used.") assert connection_objects.configured_catalog is not None @@ -333,8 +358,8 @@ def configured_catalog(connection_objects: ConnectionObjects, selected_streams: @pytest.fixture(scope="session", autouse=True) def primary_keys_per_stream( configured_catalog: ConfiguredAirbyteCatalog, -) -> Dict[str, Optional[List[str]]]: - return {stream.stream.name: stream.primary_key[0] if getattr(stream, "primary_key") else None for stream in configured_catalog.streams} +) -> dict[str, Optional[list[str]]]: + return {stream.stream.name: stream.primary_key[0] if stream.primary_key else None for stream in configured_catalog.streams} @pytest.fixture(scope="session") @@ -345,7 +370,7 @@ def configured_streams( @pytest.fixture(scope="session") -def state(connection_objects: ConnectionObjects) -> Optional[Dict]: +def state(connection_objects: ConnectionObjects) -> Optional[dict]: return connection_objects.state diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/consts.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/consts.py index 3ece4d135814..16bfc69e55bd 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/consts.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/consts.py @@ -1,3 +1,4 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations MAX_LINES_IN_REPORT = 1000 diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/report.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/report.py index 65281933e30d..3ab42032e3ba 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/report.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/report.py @@ -5,10 +5,11 @@ import datetime import json from collections import defaultdict +from collections.abc import Iterable, MutableMapping from copy import deepcopy from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, MutableMapping, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any, Optional import requests import yaml @@ -42,12 +43,12 @@ def __init__(self, path: Path, pytest_config: Config) -> None: self.secret_properties = self.get_secret_properties() self.created_at = datetime.datetime.utcnow() self.updated_at = self.created_at - self.control_execution_results_per_command: Dict[Command, ExecutionResult] = {} - self.target_execution_results_per_command: Dict[Command, ExecutionResult] = {} - self.test_results: List[Dict[str, Any]] = [] + self.control_execution_results_per_command: dict[Command, ExecutionResult] = {} + self.target_execution_results_per_command: dict[Command, ExecutionResult] = {} + self.test_results: list[dict[str, Any]] = [] self.update(ReportState.INITIALIZING) - def get_secret_properties(self) -> List: + def get_secret_properties(self) -> list: response = requests.get(self.SPEC_SECRET_MASK_URL) response.raise_for_status() return yaml.safe_load(response.text)["properties"] @@ -66,7 +67,7 @@ def add_target_execution_result(self, target_execution_result: ExecutionResult) self.update() def add_test_result(self, test_report: pytest.TestReport, test_documentation: Optional[str] = None) -> None: - cut_properties: List[Tuple[str, str]] = [] + cut_properties: list[tuple[str, str]] = [] for property_name, property_value in test_report.user_properties: if len(str(property_value).splitlines()) > MAX_LINES_IN_REPORT: cut_property_name = f"{property_name} (truncated)" @@ -141,7 +142,7 @@ def scrub_secrets_from_config(self, to_scrub: MutableMapping) -> MutableMapping: return to_scrub ### REPORT CONTENT HELPERS ### - def get_stream_coverage_metrics(self) -> Dict[str, str]: + def get_stream_coverage_metrics(self) -> dict[str, str]: configured_catalog_stream_count = ( len(self.connection_objects.configured_catalog.streams) if self.connection_objects.configured_catalog else 0 ) @@ -154,12 +155,13 @@ def get_stream_coverage_metrics(self) -> Dict[str, str]: def get_record_count_per_stream( self, - ) -> Dict[Command, Dict[str, Dict[str, int] | int]]: - record_count_per_command_and_stream: Dict[Command, Dict[str, Dict[str, int] | int]] = {} + ) -> dict[Command, dict[str, dict[str, int] | int]]: + record_count_per_command_and_stream: dict[Command, dict[str, dict[str, int] | int]] = {} for control_result, target_result in zip( self.control_execution_results_per_command.values(), self.target_execution_results_per_command.values(), + strict=False, ): per_stream_count = defaultdict(lambda: {"control": 0, "target": 0}) # type: ignore for result, source in [ @@ -176,8 +178,8 @@ def get_record_count_per_stream( return record_count_per_command_and_stream - def get_untested_streams(self) -> List[str]: - streams_with_data: Set[str] = set() + def get_untested_streams(self) -> list[str]: + streams_with_data: set[str] = set() for stream_count in self.get_record_count_per_stream().values(): streams_with_data.update(stream_count.keys()) @@ -185,7 +187,7 @@ def get_untested_streams(self) -> List[str]: return [stream.name for stream in catalog_streams if stream.name not in streams_with_data] - def get_selected_streams(self) -> Dict[str, Dict[str, SyncMode | bool]]: + def get_selected_streams(self) -> dict[str, dict[str, SyncMode | bool]]: untested_streams = self.get_untested_streams() return ( { @@ -202,16 +204,16 @@ def get_selected_streams(self) -> Dict[str, Dict[str, SyncMode | bool]]: else {} ) - def get_sync_mode_coverage(self) -> Dict[SyncMode, int]: - count_per_sync_mode: Dict[SyncMode, int] = defaultdict(int) + def get_sync_mode_coverage(self) -> dict[SyncMode, int]: + count_per_sync_mode: dict[SyncMode, int] = defaultdict(int) for s in self.get_selected_streams().values(): count_per_sync_mode[s["sync_mode"]] += 1 return count_per_sync_mode def get_message_count_per_type( self, - ) -> Tuple[List[Command], Dict[Type, Dict[Command, Dict[str, int]]]]: - message_count_per_type_and_command: Dict[Type, Dict[Command, Dict[str, int]]] = {} + ) -> tuple[list[Command], dict[Type, dict[Command, dict[str, int]]]]: + message_count_per_type_and_command: dict[Type, dict[Command, dict[str, int]]] = {} all_message_types = set() all_commands = set() # Gather all message types from both control and target execution reports @@ -251,12 +253,13 @@ def get_message_count_per_type( def get_http_metrics_per_command( self, - ) -> Dict[Command, Dict[str, Dict[str, int | str] | int]]: - metrics_per_command: Dict[Command, Dict[str, Dict[str, int | str] | int]] = {} + ) -> dict[Command, dict[str, dict[str, int | str] | int]]: + metrics_per_command: dict[Command, dict[str, dict[str, int | str] | int]] = {} for control_result, target_result in zip( self.control_execution_results_per_command.values(), self.target_execution_results_per_command.values(), + strict=False, ): control_flow_count = len(control_result.http_flows) control_all_urls = [f.request.url for f in control_result.http_flows] @@ -292,7 +295,7 @@ def get_http_metrics_per_command( def get_requested_urls_per_command( self, - ) -> Dict[Command, List[Tuple[int, str, str]]]: + ) -> dict[Command, list[tuple[int, str, str]]]: requested_urls_per_command = {} all_commands = sorted( list(set(self.control_execution_results_per_command.keys()).union(set(self.target_execution_results_per_command.keys()))), diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/stash_keys.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/stash_keys.py index b3e3d3db8be0..e5fdb8284187 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/stash_keys.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/stash_keys.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from __future__ import annotations from pathlib import Path -from typing import List, Set import pytest from live_tests.commons.models import ConnectionObjects @@ -16,13 +16,15 @@ CONTROL_VERSION = pytest.StashKey[str]() DAGGER_LOG_PATH = pytest.StashKey[Path]() DUCKDB_PATH = pytest.StashKey[Path]() -HTTP_DUMP_CACHE_VOLUMES = pytest.StashKey[List]() +HTTP_DUMP_CACHE_VOLUMES = pytest.StashKey[list]() +RUN_IN_AIRBYTE_CI = pytest.StashKey[bool]() # Running in airbyte-ci, locally or in GhA +IS_PRODUCTION_CI = pytest.StashKey[bool]() # Running in airbyte-ci in GhA IS_PERMITTED_BOOL = pytest.StashKey[bool]() PR_URL = pytest.StashKey[str]() REPORT = pytest.StashKey[Report]() RETRIEVAL_REASONS = pytest.StashKey[str]() -SELECTED_STREAMS = pytest.StashKey[Set[str]]() -SESSION_START_TIMESTAMP = pytest.StashKey[int]() +SELECTED_STREAMS = pytest.StashKey[set[str]]() +SESSION_RUN_ID = pytest.StashKey[str]() SHOULD_READ_WITH_STATE = pytest.StashKey[bool]() TARGET_VERSION = pytest.StashKey[str]() TEST_ARTIFACT_DIRECTORY = pytest.StashKey[Path]() diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py index 7b963f8a2499..b5a3b7b0573c 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py @@ -1,7 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations - -from typing import Callable +from collections.abc import Callable import pytest from airbyte_protocol.models import Status, Type # type: ignore diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py index 1c9ea0035a90..e09584b48100 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py @@ -1,7 +1,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import json -from typing import Callable, Dict, Iterable, List +from collections.abc import Callable, Iterable import pytest from _pytest.fixtures import SubRequest @@ -99,7 +100,7 @@ def get_catalog(execution_result: ExecutionResult) -> AirbyteCatalog: ) -def _get_filtered_sorted_streams(streams: Dict[str, AirbyteStream], stream_set: Iterable[str], include_target: bool) -> List[Dict]: +def _get_filtered_sorted_streams(streams: dict[str, AirbyteStream], stream_set: Iterable[str], include_target: bool) -> list[dict]: return sorted( filter( lambda x: (x["name"] in stream_set if include_target else x["name"] not in stream_set), diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py index 515b79caa26b..8cfabf84e906 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py @@ -2,7 +2,8 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, List, Optional +from collections.abc import Callable, Generator, Iterable +from typing import TYPE_CHECKING, Any, Optional import pytest from airbyte_protocol.models import AirbyteMessage # type: ignore @@ -38,7 +39,7 @@ async def _check_all_pks_are_produced_in_target_version( request: SubRequest, record_property: Callable, configured_streams: Iterable[str], - primary_keys_per_stream: Dict[str, Optional[List[str]]], + primary_keys_per_stream: dict[str, Optional[list[str]]], read_with_state_control_execution_result: ExecutionResult, read_with_state_target_execution_result: ExecutionResult, ) -> None: @@ -105,7 +106,7 @@ async def _check_record_counts( read_control_execution_result: ExecutionResult, read_target_execution_result: ExecutionResult, ) -> None: - record_count_difference_per_stream: Dict[str, Dict[str, int]] = {} + record_count_difference_per_stream: dict[str, dict[str, int]] = {} for stream_name in configured_streams: control_records_count = sum(1 for _ in read_control_execution_result.get_records_per_stream(stream_name)) target_records_count = sum(1 for _ in read_target_execution_result.get_records_per_stream(stream_name)) @@ -137,7 +138,7 @@ async def _check_all_records_are_the_same( request: SubRequest, record_property: Callable, configured_streams: Iterable[str], - primary_keys_per_stream: Dict[str, Optional[List[str]]], + primary_keys_per_stream: dict[str, Optional[list[str]]], read_control_execution_result: ExecutionResult, read_target_execution_result: ExecutionResult, ) -> None: @@ -237,7 +238,7 @@ def _check_record_schema_match( if mismatches_count > 0: pytest.fail(f"{mismatches_count} streams have mismatching schemas between control and target versions.") - @pytest.mark.with_state + @pytest.mark.with_state() async def test_record_count_with_state( self, record_property: Callable, @@ -268,7 +269,7 @@ async def test_record_count_with_state( read_with_state_target_execution_result, ) - @pytest.mark.without_state + @pytest.mark.without_state() async def test_record_count_without_state( self, record_property: Callable, @@ -299,13 +300,13 @@ async def test_record_count_without_state( read_target_execution_result, ) - @pytest.mark.with_state + @pytest.mark.with_state() async def test_all_pks_are_produced_in_target_version_with_state( self, request: SubRequest, record_property: Callable, configured_streams: Iterable[str], - primary_keys_per_stream: Dict[str, Optional[List[str]]], + primary_keys_per_stream: dict[str, Optional[list[str]]], read_with_state_control_execution_result: ExecutionResult, read_with_state_target_execution_result: ExecutionResult, ) -> None: @@ -329,13 +330,13 @@ async def test_all_pks_are_produced_in_target_version_with_state( read_with_state_target_execution_result, ) - @pytest.mark.without_state + @pytest.mark.without_state() async def test_all_pks_are_produced_in_target_version_without_state( self, request: SubRequest, record_property: Callable, configured_streams: Iterable[str], - primary_keys_per_stream: Dict[str, Optional[List[str]]], + primary_keys_per_stream: dict[str, Optional[list[str]]], read_control_execution_result: ExecutionResult, read_target_execution_result: ExecutionResult, ) -> None: @@ -359,7 +360,7 @@ async def test_all_pks_are_produced_in_target_version_without_state( read_target_execution_result, ) - @pytest.mark.with_state + @pytest.mark.with_state() async def test_record_schema_match_with_state( self, request: SubRequest, @@ -379,7 +380,7 @@ async def test_record_schema_match_with_state( read_with_state_target_execution_result, ) - @pytest.mark.without_state + @pytest.mark.without_state() async def test_record_schema_match_without_state( self, request: SubRequest, @@ -399,13 +400,13 @@ async def test_record_schema_match_without_state( read_target_execution_result, ) - @pytest.mark.with_state + @pytest.mark.with_state() async def test_all_records_are_the_same_with_state( self, request: SubRequest, record_property: Callable, configured_streams: Iterable[str], - primary_keys_per_stream: Dict[str, Optional[List[str]]], + primary_keys_per_stream: dict[str, Optional[list[str]]], read_with_state_control_execution_result: ExecutionResult, read_with_state_target_execution_result: ExecutionResult, ) -> None: @@ -430,13 +431,13 @@ async def test_all_records_are_the_same_with_state( read_with_state_target_execution_result, ) - @pytest.mark.without_state + @pytest.mark.without_state() async def test_all_records_are_the_same_without_state( self, request: SubRequest, record_property: Callable, configured_streams: Iterable[str], - primary_keys_per_stream: Dict[str, Optional[List[str]]], + primary_keys_per_stream: dict[str, Optional[list[str]]], read_control_execution_result: ExecutionResult, read_target_execution_result: ExecutionResult, ) -> None: @@ -466,9 +467,9 @@ def _get_diff_on_stream_with_pk( request: SubRequest, record_property: Callable, stream: str, - control_records: List[AirbyteMessage], - target_records: List[AirbyteMessage], - primary_key: List[str], + control_records: list[AirbyteMessage], + target_records: list[AirbyteMessage], + primary_key: list[str], ) -> Optional[Iterable[str]]: control_pks = {r.record.data[primary_key[0]] for r in control_records} target_pks = {r.record.data[primary_key[0]] for r in target_records} @@ -528,8 +529,8 @@ def _get_diff_on_stream_without_pk( request: SubRequest, record_property: Callable, stream: str, - control_records: List[AirbyteMessage], - target_records: List[AirbyteMessage], + control_records: list[AirbyteMessage], + target_records: list[AirbyteMessage], ) -> Optional[Iterable[str]]: diff = get_and_write_diff( request, @@ -546,11 +547,11 @@ def _get_diff_on_stream_without_pk( def _get_filtered_sorted_records( - records: List[AirbyteMessage], + records: list[AirbyteMessage], primary_key_set: set[Generator[Any, Any, None]], include_target: bool, - primary_key: List[str], -) -> List[Dict]: + primary_key: list[str], +) -> list[dict]: """ Get a list of records sorted by primary key, and filtered as specified. diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py index a59247e69b14..c9101651efa5 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations -from typing import Callable +from collections.abc import Callable import pytest from airbyte_protocol.models import Type # type: ignore diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/utils.py index 331a18a05b10..9862c84fcc5c 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/utils.py @@ -3,8 +3,9 @@ import json import logging +from collections.abc import Callable, Iterable from pathlib import Path -from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union import pytest from airbyte_protocol.models import AirbyteMessage, Type # type: ignore @@ -42,11 +43,11 @@ def write_string_to_test_artifact(request: SubRequest, content: str, filename: s def get_and_write_diff( request: SubRequest, - control_data: Union[List, Dict], - target_data: Union[List, Dict], + control_data: Union[list, dict], + target_data: Union[list, dict], filepath: str, ignore_order: bool, - exclude_paths: Optional[List[str]], + exclude_paths: Optional[list[str]], ) -> str: logger = get_test_logger(request) diff = DeepDiff( @@ -83,7 +84,7 @@ def get_and_write_diff( return "" -def fail_test_on_failing_execution_results(record_property: Callable, execution_results: List[ExecutionResult]) -> None: +def fail_test_on_failing_execution_results(record_property: Callable, execution_results: list[ExecutionResult]) -> None: error_messages = [] for execution_result in execution_results: if not execution_result.success: @@ -103,12 +104,12 @@ def fail_test_on_failing_execution_results(record_property: Callable, execution_ pytest.fail("\n".join(error_messages)) -def tail_file(file_path: Path, n: int = MAX_LINES_IN_REPORT) -> List[str]: - with open(file_path, "r") as f: +def tail_file(file_path: Path, n: int = MAX_LINES_IN_REPORT) -> list[str]: + with open(file_path) as f: # Move the cursor to the end of the file f.seek(0, 2) file_size = f.tell() - lines: List[str] = [] + lines: list[str] = [] read_size = min(4096, file_size) cursor = file_size - read_size diff --git a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py index d7dc61b831b0..be22da351d93 100644 --- a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py +++ b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py @@ -68,4 +68,3 @@ def test_write(tmp_path, messages, expected_writes): expected_path = Path(tmp_path / expected_file) assert expected_path.exists() content = expected_path.read_text() - assert content == expected_content diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index afda7813a8e5..374d056d8103 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -649,6 +649,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | | ------- | ---------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------| +| 4.9.0 | [#37440](https://github.com/airbytehq/airbyte/pull/37440) | Run regression tests with `airbyte-ci connectors test` | | 4.8.0 | [#37404](https://github.com/airbytehq/airbyte/pull/37404) | Accept a `git-repo-url` option on the `airbyte-ci` root command to checkout forked repo. | | 4.7.4 | [#37485](https://github.com/airbytehq/airbyte/pull/37485) | Allow java connectors to be written in kotlin. | | 4.7.3 | [#37101](https://github.com/airbytehq/airbyte/pull/37101) | Pin PyAirbyte version. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py index f7ae65bffcfc..6cddc3729849 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py @@ -39,11 +39,13 @@ async def _run(self, *args: Any) -> StepResult: await connector.with_exec(["spec"]) except ExecError as e: return StepResult( - step=self, status=StepStatus.FAILURE, stderr=str(e), stdout=f"Failed to run the spec command on the connector container for platform {platform}." + step=self, status=StepStatus.FAILURE, stderr=str(e), + stdout=f"Failed to run the spec command on the connector container for platform {platform}." ) build_results_per_platform[platform] = connector except QueryError as e: - return StepResult(step=self, status=StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}") + return StepResult(step=self, status=StepStatus.FAILURE, + stderr=f"Failed to build connector image for platform {platform}: {e}") success_message = ( f"The {self.context.connector.technical_name} docker image " f"was successfully built for platform(s) {', '.join(self.build_platforms)}" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py index 8194dfc38594..006b95d28415 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py @@ -30,6 +30,7 @@ class CONNECTOR_TEST_STEP_ID(str, Enum): CHECK_MIGRATION_CANDIDATE = "check_migration_candidate" POETRY_INIT = "poetry_init" DELETE_SETUP_PY = "delete_setup_py" + CONNECTOR_REGRESSION_TESTS = "connector_regression_tests" def __str__(self) -> str: return self.value diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index dff4f9b2a736..3756e2e1f6a9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -174,6 +174,10 @@ def updated_secrets_dir(self, updated_secrets_dir: Directory) -> None: def connector_acceptance_test_source_dir(self) -> Directory: return self.get_repo_dir("airbyte-integrations/bases/connector-acceptance-test") + @property + def live_tests_dir(self) -> Directory: + return self.get_repo_dir("airbyte-ci/connectors/live-tests") + @property def should_save_updated_secrets(self) -> bool: return self.use_remote_secrets and self.updated_secrets_dir is not None diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 41c1f629ff39..53b8d3761d78 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import shutil from typing import Dict, List import asyncclick as click @@ -10,6 +11,7 @@ from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline +from pipelines.airbyte_ci.connectors.test.steps.common import RegressionTests from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState @@ -85,6 +87,9 @@ async def test( """ if only_steps and skip_steps: raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.") + if not only_steps: + skip_steps = list(skip_steps) + skip_steps += [CONNECTOR_TEST_STEP_ID.REGRESSION_TEST] if ctx.obj["is_ci"]: fail_if_missing_docker_hub_creds(ctx) @@ -101,6 +106,7 @@ async def test( keep_steps=[CONNECTOR_TEST_STEP_ID(step_id) for step_id in only_steps], step_params=extra_params, ) + connectors_tests_contexts = [ ConnectorContext( pipeline_name=f"Testing connector {connector.technical_name}", @@ -144,6 +150,11 @@ async def test( update_global_commit_status_check_for_tests(ctx.obj, "failure") return False + finally: + if RegressionTests.regression_tests_artifacts_dir.exists(): + shutil.rmtree(RegressionTests.regression_tests_artifacts_dir) + main_logger.info(f" Test artifacts cleaned up from {RegressionTests.regression_tests_artifacts_dir}") + @ctx.call_on_close def send_commit_status_check() -> None: if ctx.obj["is_ci"]: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 177c7dc6a281..707192b9c554 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -6,6 +6,7 @@ import datetime import os +import time from abc import ABC, abstractmethod from functools import cached_property from pathlib import Path @@ -16,11 +17,13 @@ import yaml # type: ignore from dagger import Container, Directory from pipelines import hacks +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.steps.docker import SimpleDockerStep from pipelines.consts import INTERNAL_TOOL_PATHS, CIContext from pipelines.dagger.actions import secrets -from pipelines.helpers.utils import METADATA_FILE_NAME +from pipelines.dagger.containers.python import with_python_base +from pipelines.helpers.utils import METADATA_FILE_NAME, get_exec_result from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus @@ -301,3 +304,128 @@ async def _build_connector_acceptance_test(self, connector_under_test_container: ) return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) + + +class RegressionTests(Step): + """A step to run regression tests for a connector.""" + + context: ConnectorContext + title = "Regression tests" + skipped_exit_code = 5 + accept_extra_params = True + regression_tests_artifacts_dir = Path("/tmp/regression_tests_artifacts") + working_directory = "/app" + + @property + def default_params(self) -> STEP_PARAMS: + """Default pytest options. + + Returns: + dict: The default pytest options. + """ + return super().default_params | { + "-ra": [], # Show extra test summary info in the report for all but the passed tests + "--disable-warnings": [], # Disable warnings in the pytest report + "--durations": ["3"], # Show the 3 slowest tests in the report + } + + def regression_tests_command(self) -> List[str]: + return [ + "poetry", + "run", + "pytest", + "src/live_tests/regression_tests", + "--connector-image", + self.connector_image, + "--connection-id", + self.connection_id or "", + "--control-version", + self.control_version or "", + "--target-version", + self.target_version or "", + "--pr-url", + self.pr_url or "", + "--run-id", + self.run_id or "", + "--should-read-with-state", + str(self.should_read_with_state), + ] + + def __init__(self, context: ConnectorContext) -> None: + """Create a step to run regression tests for a connector. + + Args: + context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. + """ + super().__init__(context) + self.connector_image = context.docker_image.split(":")[0] + options = self.context.run_step_options.step_params.get(CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS, {}) + + self.connection_id = self.context.run_step_options.get_item_or_default(options, "connection-id", None) + self.pr_url = self.context.run_step_options.get_item_or_default(options, "pr-url", None) + + if not self.connection_id and self.pr_url: + raise ValueError("`connection-id` and `pr-url` are required to run regression tests.") + + self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest") + self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev") + self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", True) + self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time())) + + async def _run(self, connector_under_test_container: Container) -> StepResult: + """Run the regression test suite. + + Args: + connector_under_test (Container): The container holding the target connector test image. + + Returns: + StepResult: Failure or success of the regression tests with stdout and stderr. + """ + container = await self._build_regression_test_container(await connector_under_test_container.id()) + container = container.with_(hacks.never_fail_exec(self.regression_tests_command())) + regression_tests_artifacts_dir = str(self.regression_tests_artifacts_dir) + path_to_report = f"{regression_tests_artifacts_dir}/session_{self.run_id}/report.html" + await container.file(path_to_report).export(path_to_report) + exit_code, stdout, stderr = await get_exec_result(container) + + with open(path_to_report, "r") as fp: + regression_test_report = fp.read() + + return StepResult( + step=self, + status=self.get_step_status_from_exit_code(exit_code), + stderr=stderr, + stdout=stdout, + output=container, + report=regression_test_report, + ) + + async def _build_regression_test_container(self, target_container_id: str) -> Container: + """Create a container to run regression tests.""" + container = with_python_base(self.context) + + container = ( + ( + container.with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "install", "-y", "git", "openssh-client", "curl", "docker.io"]) + .with_exec(["bash", "-c", "curl https://sdk.cloud.google.com | bash"]) + .with_env_variable("PATH", "/root/google-cloud-sdk/bin:$PATH", expand=True) + .with_mounted_file("/root/.ssh/id_rsa", self.dagger_client.host().file(str(Path("~/.ssh/id_rsa").expanduser()))) # TODO + .with_mounted_file( + "/root/.ssh/known_hosts", self.dagger_client.host().file(str(Path("~/.ssh/known_hosts").expanduser())) # TODO + ) + .with_mounted_file( + "/root/.config/gcloud/application_default_credentials.json", + self.dagger_client.host().file(str(Path("~/.config/gcloud/application_default_credentials.json").expanduser())), # TODO + ) + .with_mounted_directory("/app", self.context.live_tests_dir) + .with_workdir("/app") + .with_exec(["pip", "install", "poetry"]) + .with_exec(["poetry", "lock", "--no-update"]) + .with_exec(["poetry", "install"]) + ) + .with_unix_socket("/var/run/docker.sock", self.dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_env_variable("RUN_IN_AIRBYTE_CI", "1") + .with_new_file("/tmp/container_id.txt", contents=str(target_container_id)) + ) + return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 538b07f0c339..9d8d512ea455 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -15,7 +15,7 @@ from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests +from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, RegressionTests from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.dagger.actions import secrets from pipelines.dagger.actions.python.poetry import with_poetry @@ -280,4 +280,12 @@ def get_test_steps(context: ConnectorContext) -> STEP_TREE: depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], ), ], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.CONNECTOR_REGRESSION_TESTS, + step=RegressionTests(context), + args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + ], ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 index 53879e94c591..e73fac6519ab 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 @@ -176,6 +176,9 @@ function copyToClipBoard(htmlElement) { {% endif %}
{{ step_result.stdout|e }}@@ -184,6 +187,7 @@ function copyToClipBoard(htmlElement) { Standard error():
{{ step_result.stderr|e }}{% endif %} + {% endif %}