From f5e170453a0ab52456c2c3203a7c571c7d3d95f2 Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:48:27 +1200 Subject: [PATCH] :sparkles: XpySTACAssetReader for reading COG, NetCDF & Zarr STAC assets (#87) An iterable-style DataPipe for STAC asset data like Cloud-Optimized GeoTIFFs, NetCDF and Zarr! Uses xarray.open_dataset(..., engine="stac") enabled by xpystac for reading STAC assets into an xarray.Dataset object. * :heavy_plus_sign: Add xpystac Extend xarray.open_dataset to accept pystac objects! * :heavy_plus_sign: Add zarr An implementation of chunked, compressed, N-dimensional arrays for Python! * :sparkles: XpySTACAssetReader for reading COG and Zarr STAC assets An iterable-style DataPipe for STAC asset data like Cloud-Optimized GeoTIFFs and Zarr! Uses xpystac for the I/O. Included a doctest and unit test, added a new section in the API docs and some more intersphinx mappings. * :green_heart: Install xpystac and zarr in docs extras Include xpystac and zarr in the 'docs' extras dependencies to fix ReadtheDocs build. * :heavy_plus_sign: Add aiohttp to dev dependencies Async http client/server framework (asyncio)! Needed to fix `ModuleNotFoundError: No module named 'aiohttp'` when accessing NetCDF files from https://nasagddp.blob.core.windows.net/nex-gddp-cmip6-references/ACCESS-CM2_historical.json * :memo: Add xpystac and zarr to pip install zen3geo[raster] docs Mention that xpystac and zarr is installed with the 'raster' extras on the main index.md page. Also mentioned that NetCDF files can be read, and added some blank lines in the XpySTACAssetReader doctest example. * :arrow_up: Bump aiohttp from 3.8.1 to 3.8.4 Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.8.1 to 3.8.4. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.8.1...v3.8.4) * :white_check_mark: Unit test for a Zarr-backed STAC asset Ensure that a STAC asset pointing to a Zarr file can be loaded using XpySTACAssetReader. Using the Daymet Annual Hawaii STAC Collection at https://planetarycomputer.microsoft.com/dataset/daymet-annual-hi for this unit test. Also edited previous unit test to specify that it is for a Cloud-Optimized GeoTIFF STAC Asset. * :wrench: Put xpystac under the stac extras Decided that xpystac fits better under the 'stac' extras, because it depends on just pystac and xarray, and has a somewhat optional dependency on stackstac. This enables a more streamlined I/O option for reading STAC Assets into an xarray.Dataset. Note that Zarr is kept under the 'raster' extras. * :pencil2: Fix NetCDF link and type hints Need to use trailing underscores for RST-style URLs, and remove the `pystac.Asset` type hint so that zen3geo works without `pystac` being installed. --- docs/_config.yml | 3 + docs/api.md | 9 + docs/index.md | 4 +- poetry.lock | 279 +++++++++++++++++------- pyproject.toml | 15 +- zen3geo/datapipes/__init__.py | 3 + zen3geo/datapipes/xpystac.py | 142 ++++++++++++ zen3geo/tests/test_datapipes_xpystac.py | 67 ++++++ 8 files changed, 433 insertions(+), 89 deletions(-) create mode 100644 zen3geo/datapipes/xpystac.py create mode 100644 zen3geo/tests/test_datapipes_xpystac.py diff --git a/docs/_config.yml b/docs/_config.yml index c9a363b..b81c2e9 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -96,6 +96,9 @@ sphinx: xbatcher: - 'https://xbatcher.readthedocs.io/en/latest/' - null + zarr: + - 'https://zarr.readthedocs.io/en/latest/' + - null extra_extensions: - 'sphinx.ext.autodoc' - 'sphinx.ext.intersphinx' diff --git a/docs/api.md b/docs/api.md index 67d8079..a8e4b32 100644 --- a/docs/api.md +++ b/docs/api.md @@ -82,3 +82,12 @@ .. autoclass:: zen3geo.datapipes.xbatcher.XbatcherSlicerIterDataPipe :show-inheritance: ``` + +### XpySTAC + +```{eval-rst} +.. automodule:: zen3geo.datapipes.xpystac +.. autoclass:: zen3geo.datapipes.XpySTACAssetReader +.. autoclass:: zen3geo.datapipes.xpystac.XpySTACAssetReaderIterDataPipe + :show-inheritance: +``` diff --git a/docs/index.md b/docs/index.md index 4bd6a69..fa36bc2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,9 +7,9 @@ Get what you need, not more, not less: | Command | Dependencies | |:-------------------------------|---------------| | `pip install zen3geo` | rioxarray, torchdata | -| `pip install zen3geo[raster]` | rioxarray, torchdata, xbatcher | +| `pip install zen3geo[raster]` | rioxarray, torchdata, xbatcher, zarr | | `pip install zen3geo[spatial]` | rioxarray, torchdata, datashader, spatialpandas | -| `pip install zen3geo[stac]` | rioxarray, torchdata, pystac, pystac-client, stackstac | +| `pip install zen3geo[stac]` | rioxarray, torchdata, pystac, pystac-client, stackstac, xpystac | | `pip install zen3geo[vector]` | rioxarray, torchdata, pyogrio[geopandas] | Retrieve more ['extras'](https://github.com/weiji14/zen3geo/blob/main/pyproject.toml) using diff --git a/poetry.lock b/poetry.lock index 24e765c..e77fb03 100644 --- a/poetry.lock +++ b/poetry.lock @@ -54,91 +54,106 @@ test = ["coveralls", "flake8", "pydocstyle", "pytest (>=4.6)", "pytest-cov"] [[package]] name = "aiohttp" -version = "3.8.1" +version = "3.8.4" description = "Async http client/server framework (asyncio)" category = "main" -optional = true +optional = false python-versions = ">=3.6" files = [ - {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1ed0b6477896559f17b9eaeb6d38e07f7f9ffe40b9f0f9627ae8b9926ae260a8"}, - {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7dadf3c307b31e0e61689cbf9e06be7a867c563d5a63ce9dca578f956609abf8"}, - {file = "aiohttp-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a79004bb58748f31ae1cbe9fa891054baaa46fb106c2dc7af9f8e3304dc30316"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12de6add4038df8f72fac606dff775791a60f113a725c960f2bab01d8b8e6b15"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f0d5f33feb5f69ddd57a4a4bd3d56c719a141080b445cbf18f238973c5c9923"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eaba923151d9deea315be1f3e2b31cc39a6d1d2f682f942905951f4e40200922"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:099ebd2c37ac74cce10a3527d2b49af80243e2a4fa39e7bce41617fbc35fa3c1"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e5d962cf7e1d426aa0e528a7e198658cdc8aa4fe87f781d039ad75dcd52c516"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61bfc23df345d8c9716d03717c2ed5e27374e0fe6f659ea64edcd27b4b044cf7"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:31560d268ff62143e92423ef183680b9829b1b482c011713ae941997921eebc8"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97ef77eb6b044134c0b3a96e16abcb05ecce892965a2124c566af0fd60f717e2"}, - {file = "aiohttp-3.8.1-cp310-cp310-win32.whl", hash = "sha256:c2aef4703f1f2ddc6df17519885dbfa3514929149d3ff900b73f45998f2532fa"}, - {file = "aiohttp-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:713ac174a629d39b7c6a3aa757b337599798da4c1157114a314e4e391cd28e32"}, - {file = "aiohttp-3.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:473d93d4450880fe278696549f2e7aed8cd23708c3c1997981464475f32137db"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b5eeae8e019e7aad8af8bb314fb908dd2e028b3cdaad87ec05095394cce632"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af642b43ce56c24d063325dd2cf20ee012d2b9ba4c3c008755a301aaea720ad"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3630c3ef435c0a7c549ba170a0633a56e92629aeed0e707fec832dee313fb7a"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a4a4e30bf1edcad13fb0804300557aedd07a92cabc74382fdd0ba6ca2661091"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6f8b01295e26c68b3a1b90efb7a89029110d3a4139270b24fda961893216c440"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a25fa703a527158aaf10dafd956f7d42ac6d30ec80e9a70846253dd13e2f067b"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:5bfde62d1d2641a1f5173b8c8c2d96ceb4854f54a44c23102e2ccc7e02f003ec"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:51467000f3647d519272392f484126aa716f747859794ac9924a7aafa86cd411"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:03a6d5349c9ee8f79ab3ff3694d6ce1cfc3ced1c9d36200cb8f08ba06bd3b782"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:102e487eeb82afac440581e5d7f8f44560b36cf0bdd11abc51a46c1cd88914d4"}, - {file = "aiohttp-3.8.1-cp36-cp36m-win32.whl", hash = "sha256:4aed991a28ea3ce320dc8ce655875e1e00a11bdd29fe9444dd4f88c30d558602"}, - {file = "aiohttp-3.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b0e20cddbd676ab8a64c774fefa0ad787cc506afd844de95da56060348021e96"}, - {file = "aiohttp-3.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:37951ad2f4a6df6506750a23f7cbabad24c73c65f23f72e95897bb2cecbae676"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c23b1ad869653bc818e972b7a3a79852d0e494e9ab7e1a701a3decc49c20d51"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15b09b06dae900777833fe7fc4b4aa426556ce95847a3e8d7548e2d19e34edb8"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:477c3ea0ba410b2b56b7efb072c36fa91b1e6fc331761798fa3f28bb224830dd"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2f2f69dca064926e79997f45b2f34e202b320fd3782f17a91941f7eb85502ee2"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef9612483cb35171d51d9173647eed5d0069eaa2ee812793a75373447d487aa4"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6d69f36d445c45cda7b3b26afef2fc34ef5ac0cdc75584a87ef307ee3c8c6d00"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:55c3d1072704d27401c92339144d199d9de7b52627f724a949fc7d5fc56d8b93"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d00268fcb9f66fbcc7cd9fe423741d90c75ee029a1d15c09b22d23253c0a44"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:07b05cd3305e8a73112103c834e91cd27ce5b4bd07850c4b4dbd1877d3f45be7"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c34dc4958b232ef6188c4318cb7b2c2d80521c9a56c52449f8f93ab7bc2a8a1c"}, - {file = "aiohttp-3.8.1-cp37-cp37m-win32.whl", hash = "sha256:d2f9b69293c33aaa53d923032fe227feac867f81682f002ce33ffae978f0a9a9"}, - {file = "aiohttp-3.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6ae828d3a003f03ae31915c31fa684b9890ea44c9c989056fea96e3d12a9fa17"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0c7ebbbde809ff4e970824b2b6cb7e4222be6b95a296e46c03cf050878fc1785"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b7ef7cbd4fec9a1e811a5de813311ed4f7ac7d93e0fda233c9b3e1428f7dd7b"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c3d6a4d0619e09dcd61021debf7059955c2004fa29f48788a3dfaf9c9901a7cd"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718626a174e7e467f0558954f94af117b7d4695d48eb980146016afa4b580b2e"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:589c72667a5febd36f1315aa6e5f56dd4aa4862df295cb51c769d16142ddd7cd"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ed076098b171573161eb146afcb9129b5ff63308960aeca4b676d9d3c35e700"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:086f92daf51a032d062ec5f58af5ca6a44d082c35299c96376a41cbb33034675"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:11691cf4dc5b94236ccc609b70fec991234e7ef8d4c02dd0c9668d1e486f5abf"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:31d1e1c0dbf19ebccbfd62eff461518dcb1e307b195e93bba60c965a4dcf1ba0"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11a67c0d562e07067c4e86bffc1553f2cf5b664d6111c894671b2b8712f3aba5"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:bb01ba6b0d3f6c68b89fce7305080145d4877ad3acaed424bae4d4ee75faa950"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44db35a9e15d6fe5c40d74952e803b1d96e964f683b5a78c3cc64eb177878155"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:844a9b460871ee0a0b0b68a64890dae9c415e513db0f4a7e3cab41a0f2fedf33"}, - {file = "aiohttp-3.8.1-cp38-cp38-win32.whl", hash = "sha256:7d08744e9bae2ca9c382581f7dce1273fe3c9bae94ff572c3626e8da5b193c6a"}, - {file = "aiohttp-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:04d48b8ce6ab3cf2097b1855e1505181bdd05586ca275f2505514a6e274e8e75"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a996d01ca39b8dfe77440f3cd600825d05841088fd6bc0144cc6c2ec14cc5f74"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:13487abd2f761d4be7c8ff9080de2671e53fff69711d46de703c310c4c9317ca"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea302f34477fda3f85560a06d9ebdc7fa41e82420e892fc50b577e35fc6a50b2"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f635ce61a89c5732537a7896b6319a8fcfa23ba09bec36e1b1ac0ab31270d2"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e999f2d0e12eea01caeecb17b653f3713d758f6dcc770417cf29ef08d3931421"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0770e2806a30e744b4e21c9d73b7bee18a1cfa3c47991ee2e5a65b887c49d5cf"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c7cefb4b0640703eb1069835c02486669312bf2f12b48a748e0a7756d0de33d"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:71927042ed6365a09a98a6377501af5c9f0a4d38083652bcd2281a06a5976724"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:28d490af82bc6b7ce53ff31337a18a10498303fe66f701ab65ef27e143c3b0ef"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:b6613280ccedf24354406caf785db748bebbddcf31408b20c0b48cb86af76866"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81e3d8c34c623ca4e36c46524a3530e99c0bc95ed068fd6e9b55cb721d408fb2"}, - {file = "aiohttp-3.8.1-cp39-cp39-win32.whl", hash = "sha256:7187a76598bdb895af0adbd2fb7474d7f6025d170bc0a1130242da817ce9e7d1"}, - {file = "aiohttp-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:1c182cb873bc91b411e184dab7a2b664d4fea2743df0e4d57402f7f3fa644bac"}, - {file = "aiohttp-3.8.1.tar.gz", hash = "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, + {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, + {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, + {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, + {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, + {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, + {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, + {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, + {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, + {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, + {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, + {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, ] [package.dependencies] aiosignal = ">=1.1.2" async-timeout = ">=4.0.0a3,<5.0" attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<3.0" +charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" @@ -151,7 +166,7 @@ name = "aiosignal" version = "1.2.0" description = "aiosignal: a list of registered asynchronous callbacks" category = "main" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, @@ -185,6 +200,17 @@ files = [ {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, ] +[[package]] +name = "asciitree" +version = "0.3.3" +description = "Draws ASCII trees." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"}, +] + [[package]] name = "asttokens" version = "2.0.5" @@ -208,7 +234,7 @@ name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" category = "main" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, @@ -913,6 +939,18 @@ files = [ {file = "executing-0.8.3.tar.gz", hash = "sha256:c6554e21c6b060590a6d3be4b82fb78f8f0194d809de5ea7df1c093763311501"}, ] +[[package]] +name = "fasteners" +version = "0.18" +description = "A python package that provides useful locks" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "fasteners-0.18-py3-none-any.whl", hash = "sha256:1d4caf5f8db57b0e4107d94fd5a1d02510a450dced6ca77d1839064c1bacf20c"}, + {file = "fasteners-0.18.tar.gz", hash = "sha256:cb7c13ef91e0c7e4fe4af38ecaf6b904ec3f5ce0dda06d34924b6b74b869d953"}, +] + [[package]] name = "fastjsonschema" version = "2.15.3" @@ -1020,7 +1058,7 @@ name = "frozenlist" version = "1.3.1" description = "A list-like structure which implements collections.abc.MutableSequence" category = "main" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f271c93f001748fc26ddea409241312a75e13466b06c94798d1a341cf0e6989"}, @@ -2122,7 +2160,7 @@ name = "multidict" version = "6.0.2" description = "multidict implementation" category = "main" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, @@ -2407,6 +2445,39 @@ llvmlite = ">=0.39.0dev0,<0.40" numpy = ">=1.18,<1.24" setuptools = "*" +[[package]] +name = "numcodecs" +version = "0.11.0" +description = "A Python package providing buffer compression and transformation codecs for use" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ + {file = "numcodecs-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bc116752be45b4f9dca4315e5a2b4185e3b46f68c997dbb84aef334ceb5a1d"}, + {file = "numcodecs-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c27dfca402f69fbfa01c46fb572086e77f38121192160cc8ed1177dc30702c52"}, + {file = "numcodecs-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:0fabc7dfdf64a9555bf8a34911e05b415793c67a1377207dc79cd96342291fa1"}, + {file = "numcodecs-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dae3f5678f247336c84e7315a0c59a4fec7c33eb7db72d78ff5c776479a812e"}, + {file = "numcodecs-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32697785b786bb0039d3feeaabdc10f25eda6c149700cde954653aaa47637832"}, + {file = "numcodecs-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:8c2f36b21162c6ebccc05d3fe896f86b91dcf8709946809f730cc23a37f8234d"}, + {file = "numcodecs-0.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c240858bf29e0ff254b1db60430e8b2658b8c8328b684f80033289d94807a7c"}, + {file = "numcodecs-0.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee5bda16e9d26a7a39fc20b6c1cec23b4debc314df5cfae3ed505149c2eeafc4"}, + {file = "numcodecs-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:bd05cdb853c7bcfde2efc809a9df2c5e205b96f70405b810e5788b45d0d81f73"}, + {file = "numcodecs-0.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:694dc2e80b1f169b7deb14bdd0a04b20e5f17ef32cb0f81b71ab690406ec6bd9"}, + {file = "numcodecs-0.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf3925eeb37aed0e6c04d7fb9614133a3c8426dc77f8bda54c99c601a44b3bd3"}, + {file = "numcodecs-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:11596b71267417425ea8afb407477a67d684f434c8b07b1dd59c25a97d5c3ccb"}, + {file = "numcodecs-0.11.0.tar.gz", hash = "sha256:6c058b321de84a1729299b0eae4d652b2e48ea1ca7f9df0da65cb13470e635eb"}, +] + +[package.dependencies] +entrypoints = "*" +numpy = ">=1.7" + +[package.extras] +docs = ["mock", "numpydoc", "sphinx", "sphinx-issues"] +msgpack = ["msgpack"] +test = ["coverage", "flake8", "pytest", "pytest-cov"] +zfpy = ["zfpy (>=1.0.0)"] + [[package]] name = "numpy" version = "1.23.5" @@ -4833,6 +4904,25 @@ dev = ["adlfs", "asv", "coverage", "pytest", "pytest-cov", "tensorflow", "torch" tensorflow = ["tensorflow"] torch = ["torch"] +[[package]] +name = "xpystac" +version = "0.0.1" +description = "Extend xarray.open_dataset to accept pystac objects" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ + {file = "xpystac-0.0.1-py3-none-any.whl", hash = "sha256:89d68e5ba9135f43f78991cb4dd9dd3cdc5802aefdfc0cf9458bf080257956aa"}, + {file = "xpystac-0.0.1.tar.gz", hash = "sha256:5964e24dcb140efa8854782713c86a3a566a78d2cd4b14fb377d19866e308005"}, +] + +[package.dependencies] +pystac = ">=1.0.0b3" +xarray = "*" + +[package.extras] +dev = ["pre-commit", "pytest"] + [[package]] name = "xyzservices" version = "2022.6.0" @@ -4850,7 +4940,7 @@ name = "yarl" version = "1.8.1" description = "Yet another URL library" category = "main" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, @@ -4918,6 +5008,27 @@ files = [ idna = ">=2.0" multidict = ">=4.0" +[[package]] +name = "zarr" +version = "2.14.2" +description = "An implementation of chunked, compressed, N-dimensional arrays for Python" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ + {file = "zarr-2.14.2-py3-none-any.whl", hash = "sha256:feb72d6ccc43c447661861020dbf2e685a8367e80d890b6f5905954400394ed2"}, + {file = "zarr-2.14.2.tar.gz", hash = "sha256:68ec59b8ebdfc4fee5e32bd6c0ce273f72f7d4ed017454b45327c673c60907bc"}, +] + +[package.dependencies] +asciitree = "*" +fasteners = "*" +numcodecs = ">=0.10.0" +numpy = ">=1.20" + +[package.extras] +jupyter = ["ipytree (>=0.2.2)", "ipywidgets (>=8.0.0)", "notebook"] + [[package]] name = "zict" version = "2.2.0" @@ -4950,13 +5061,13 @@ docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -docs = ["adlfs", "contextily", "datashader", "graphviz", "jupyter-book", "matplotlib", "planetary-computer", "pyogrio", "pystac", "pystac-client", "stackstac", "spatialpandas", "xbatcher"] -raster = ["xbatcher"] +docs = ["adlfs", "contextily", "datashader", "graphviz", "jupyter-book", "matplotlib", "planetary-computer", "pyogrio", "pystac", "pystac-client", "stackstac", "spatialpandas", "xbatcher", "xpystac", "zarr"] +raster = ["xbatcher", "zarr"] spatial = ["datashader", "spatialpandas"] -stac = ["pystac", "pystac-client", "stackstac"] +stac = ["pystac", "pystac-client", "stackstac", "xpystac"] vector = ["pyogrio"] [metadata] lock-version = "2.0" python-versions = ">=3.8, <4.0" -content-hash = "7f1829e82d7c54944a60da872a89e7b454bd1e766300f0895d3538c450da9ea5" +content-hash = "90dfbece41b0ddeecc3b882e82574ef0ac8c40b51fca9c7e201343d92595e4d1" diff --git a/pyproject.toml b/pyproject.toml index 92b6f52..789cb7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,8 @@ pystac-client = {version = ">=0.4.0", optional = true} spatialpandas = {version = ">=0.4.0", optional = true} stackstac = {version = ">=0.4.0", optional = true} xbatcher = {version = ">=0.2.0", optional = true} +xpystac = {version = ">=0.0.1", optional = true} +zarr = {version = ">=2.13.0", optional = true} # Docs adlfs = {version = "*", optional = true} contextily = {version = "*", optional = true} @@ -50,6 +52,7 @@ matplotlib = {version = "*", optional = true} planetary-computer = {version="*", optional=true} [tool.poetry.group.dev.dependencies] +aiohttp = "*" black = "*" pytest = "*" @@ -67,9 +70,14 @@ docs = [ "pystac_client", "stackstac", "spatialpandas", - "xbatcher" + "xbatcher", + "xpystac", + "zarr" +] +raster = [ + "xbatcher", + "zarr" ] -raster = ["xbatcher"] spatial = [ "datashader", "spatialpandas" @@ -77,7 +85,8 @@ spatial = [ stac = [ "pystac", "pystac_client", - "stackstac" + "stackstac", + "xpystac" ] vector = ["pyogrio"] diff --git a/zen3geo/datapipes/__init__.py b/zen3geo/datapipes/__init__.py index a5f9b61..46e36e0 100644 --- a/zen3geo/datapipes/__init__.py +++ b/zen3geo/datapipes/__init__.py @@ -20,3 +20,6 @@ StackSTACStackerIterDataPipe as StackSTACStacker, ) from zen3geo.datapipes.xbatcher import XbatcherSlicerIterDataPipe as XbatcherSlicer +from zen3geo.datapipes.xpystac import ( + XpySTACAssetReaderIterDataPipe as XpySTACAssetReader, +) diff --git a/zen3geo/datapipes/xpystac.py b/zen3geo/datapipes/xpystac.py new file mode 100644 index 0000000..ae0046f --- /dev/null +++ b/zen3geo/datapipes/xpystac.py @@ -0,0 +1,142 @@ +""" +DataPipes for `xpystac `__. +""" +from typing import Any, Dict, Iterator, Optional + +import xarray as xr + +try: + import pystac + import xpystac +except ImportError: + pystac = None + xpystac = None +from torchdata.datapipes import functional_datapipe +from torchdata.datapipes.iter import IterDataPipe +from torchdata.datapipes.utils import StreamWrapper + + +@functional_datapipe("read_from_xpystac") +class XpySTACAssetReaderIterDataPipe(IterDataPipe[StreamWrapper]): + """ + Takes a :py:class:`pystac.Asset` object containing raster data (e.g. + :doc:`Zarr `, + `NetCDF `__, + `Cloud-Optimized GeoTIFF `__, etc) from local disk + or URLs (as long as they can be read by xpystac) and yields + :py:class:`xarray.Dataset` objects (functional name: + ``read_from_xpystac``). + + Based on + https://github.com/pytorch/data/blob/v0.5.1/torchdata/datapipes/iter/load/iopath.py#L42-L97 + + Parameters + ---------- + source_datapipe : IterDataPipe[pystac.Asset] + A DataPipe that contains :py:class:`pystac.Asset` objects to raster + files such as :doc:`Zarr `, + `NetCDF `__, + `Cloud-Optimized GeoTIFF `__, etc. + + engine : str or xarray.backends.BackendEntrypoint + Engine to use when reading files. If not provided, the default engine + will be the "stac" backend from ``xpystac``. Alternatively, set + ``engine=None`` to let ``xarray`` choose the default engine based on + available dependencies, with a preference for "netcdf4". See also + :py:func:`xarray.open_dataset` for details about other engine options. + + kwargs : Optional + Extra keyword arguments to pass to :py:func:`xarray.open_dataset`. + + Yields + ------ + stream_obj : xarray.Dataset + A :py:class:`xarray.Dataset` object containing the raster data. + + Raises + ------ + ModuleNotFoundError + If ``xpystac`` is not installed. See + `install instructions for xpystac + `__, + (e.g. via ``pip install xpystac``) before using this class. + + Example + ------- + >>> import pytest + >>> pystac = pytest.importorskip("pystac") + >>> xpystac = pytest.importorskip("xpystac") + >>> zarr = pytest.importorskip("zarr") + ... + >>> from torchdata.datapipes.iter import IterableWrapper + >>> from zen3geo.datapipes import XpySTACAssetReader + ... + >>> # Read in STAC Asset using DataPipe + >>> collection_url: str = "https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasa-nex-gddp-cmip6" + >>> asset: pystac.Asset = pystac.Collection.from_file(href=collection_url).assets[ + ... "ACCESS-CM2.historical" + ... ] + >>> dp = IterableWrapper(iterable=[asset]) + >>> dp_xpystac = dp.read_from_xpystac() + ... + >>> # Loop or iterate over the DataPipe stream + >>> it = iter(dp_xpystac) + >>> dataset = next(it) + >>> dataset.sizes + Frozen({'time': 23741, 'lat': 600, 'lon': 1440}) + >>> print(dataset.data_vars) + Data variables: + hurs (time, lat, lon) float32 ... + huss (time, lat, lon) float32 ... + pr (time, lat, lon) float32 ... + rlds (time, lat, lon) float32 ... + rsds (time, lat, lon) float32 ... + sfcWind (time, lat, lon) float32 ... + tas (time, lat, lon) float32 ... + tasmax (time, lat, lon) float32 ... + tasmin (time, lat, lon) float32 ... + >>> dataset.attrs # doctest: +NORMALIZE_WHITESPACE + {'Conventions': 'CF-1.7', + 'activity': 'NEX-GDDP-CMIP6', + 'cmip6_institution_id': 'CSIRO-ARCCSS', + 'cmip6_license': 'CC-BY-SA 4.0', + 'cmip6_source_id': 'ACCESS-CM2', + ... + 'history': '2021-10-04T13:59:21.654137+00:00: install global attributes', + 'institution': 'NASA Earth Exchange, NASA Ames Research Center, ... + 'product': 'output', + 'realm': 'atmos', + 'references': 'BCSD method: Thrasher et al., 2012, ... + 'resolution_id': '0.25 degree', + 'scenario': 'historical', + 'source': 'BCSD', + 'title': 'ACCESS-CM2, r1i1p1f1, historical, global downscaled CMIP6 ... + 'tracking_id': '16d27564-470f-41ea-8077-f4cc3efa5bfe', + 'variant_label': 'r1i1p1f1', + 'version': '1.0'} + """ + + def __init__( + self, + source_datapipe: IterDataPipe, + engine: str = "stac", + **kwargs: Optional[Dict[str, Any]] + ) -> None: + if xpystac is None: + raise ModuleNotFoundError( + "Package `xpystac` is required to be installed to use this datapipe. " + "Please use `pip install xpystac` " + "to install the package" + ) + self.source_datapipe: IterDataPipe = source_datapipe + self.engine: str = engine + self.kwargs = kwargs + + def __iter__(self) -> Iterator[StreamWrapper]: + for asset in self.source_datapipe: + yield StreamWrapper( + xr.open_dataset(asset, engine=self.engine, **self.kwargs) + ) + + def __len__(self) -> int: + return len(self.source_datapipe) diff --git a/zen3geo/tests/test_datapipes_xpystac.py b/zen3geo/tests/test_datapipes_xpystac.py new file mode 100644 index 0000000..4d54840 --- /dev/null +++ b/zen3geo/tests/test_datapipes_xpystac.py @@ -0,0 +1,67 @@ +""" +Tests for pystac datapipes. +""" +import pytest +from torchdata.datapipes.iter import IterableWrapper + +from zen3geo.datapipes import XpySTACAssetReader + +pystac = pytest.importorskip("pystac") +xpystac = pytest.importorskip("xpystac") + + +# %% +def test_xpystac_asset_reader_cog(): + """ + Ensure that XpySTACAssetReader works to read in a pystac.Asset object + stored as a Cloud-Optimized GeoTIFF and output to an xarray.Dataset object. + """ + item_url: str = "https://github.com/stac-utils/pystac/raw/v1.7.1/tests/data-files/raster/raster-sentinel2-example.json" + asset: pystac.Asset = pystac.Item.from_file(href=item_url).assets["overview"] + assert asset.media_type == pystac.MediaType.COG + + dp = IterableWrapper(iterable=[asset]) + + # Using class constructors + dp_xpystac = XpySTACAssetReader(source_datapipe=dp) + # Using functional form (recommended) + dp_xpystac = dp.read_from_xpystac() + + assert len(dp_xpystac) == 1 + it = iter(dp_xpystac) + dataset = next(it) + + assert dataset.sizes == {"band": 3, "x": 343, "y": 343} + assert dataset.band_data.dtype == "float32" + assert dataset.rio.bounds() == (399960.0, 4090240.0, 509720.0, 4200000.0) + assert dataset.rio.resolution() == (320.0, -320.0) + assert dataset.rio.crs == "EPSG:32633" + + +def test_xpystac_asset_reader_zarr(): + """ + Ensure that XpySTACAssetReader works to read in a pystac.Asset object + stored as a Zarr file and output to an xarray.Dataset object. + """ + collection_url: str = "https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-daily-hi" + asset: pystac.Asset = pystac.Collection.from_file(href=collection_url).assets[ + "zarr-https" + ] + assert asset.media_type == "application/vnd+zarr" + + dp = IterableWrapper(iterable=[asset]) + + # Using class constructors + dp_xpystac = XpySTACAssetReader(source_datapipe=dp) + # Using functional form (recommended) + dp_xpystac = dp.read_from_xpystac() + + assert len(dp_xpystac) == 1 + it = iter(dp_xpystac) + dataset = next(it) + + assert dataset.sizes == {"time": 14965, "y": 584, "x": 284, "nv": 2} + assert dataset.prcp.dtype == "float32" + assert dataset.rio.bounds() == (-5802750.0, -622500.0, -5518750.0, -38500.0) + assert dataset.rio.resolution() == (1000.0, -1000.0) + assert dataset.rio.grid_mapping == "lambert_conformal_conic"