From dd76de389082c8a231b25ac201167cae604088be Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 25 May 2022 23:27:38 -0400 Subject: [PATCH] bump minimum python versions (#135) * bump python versions * fix test * update ci envs and tutorial * fix docs warnings and readme * fix ci upstream env * speed up CI with caching, mamba * ignore lack of PDF driver Co-authored-by: Scott Henderson --- .binder/environment.yml | 2 +- .github/workflows/main.yaml | 27 ++++-- .pre-commit-config.yaml | 6 +- README.md | 20 +++-- ...vironment-3.7.yml => environment-3.10.yml} | 2 +- ci/environment-dev.yml | 18 +++- ci/environment-docs.yml | 4 +- ci/environment-gui.yml | 57 ------------ ci/environment-upstream.yml | 9 +- docs/source/contributing.rst | 4 +- docs/source/tutorial.rst | 90 +++++++++---------- intake_stac/catalog.py | 2 +- intake_stac/tests/test_catalog.py | 5 +- setup.py | 2 +- 14 files changed, 108 insertions(+), 140 deletions(-) rename ci/{environment-3.7.yml => environment-3.10.yml} (91%) delete mode 100644 ci/environment-gui.yml diff --git a/.binder/environment.yml b/.binder/environment.yml index 86bdd42..bc2b2d8 100644 --- a/.binder/environment.yml +++ b/.binder/environment.yml @@ -28,7 +28,7 @@ dependencies: - panel - pip - pyproj - - python=3.8 + - python=3.9 - pyyaml - pystac - pytest diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c17a3cb..fe14d52 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -13,18 +13,35 @@ jobs: strategy: fail-fast: false matrix: - CONDA_ENV: [3.7, 3.8, 3.9, upstream] + CONDA_ENV: ["3.8", "3.9", "3.10", "upstream"] steps: - name: Checkout - uses: actions/checkout@v2.4.0 + uses: actions/checkout@v3 - - name: Setup Miniconda + - name: Cache Conda Packages + uses: actions/cache@v3 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: conda-${{ env.CACHE_NUMBER }}-${{hashFiles(format('ci/environment-{0}.yml',matrix.CONDA_ENV)) }} + + - name: Cache Pip Packages (upstream environment) + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('ci/environment-upstream.yml') }} + + - name: Setup Miniconda (Mambaforge) uses: conda-incubator/setup-miniconda@v2.1.1 with: - auto-update-conda: true - auto-activate-base: false + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true activate-environment: intake-stac environment-file: ci/environment-${{ matrix.CONDA_ENV }}.yml + use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - name: Development Install Intake-STAC shell: bash -l {0} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d06469a..a5c2612 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -10,7 +10,7 @@ repos: - id: double-quote-string-fixer - repo: https://github.com/ambv/black - rev: 22.1.0 + rev: 22.3.0 hooks: - id: black args: ["--line-length", "100", "--skip-string-normalization"] @@ -29,6 +29,6 @@ repos: - id: isort - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.1 + rev: v2.6.2 hooks: - id: prettier diff --git a/README.md b/README.md index 4a5a7dc..f5cd008 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,23 @@ $ pip install git+https://github.com/intake/intake-stac ## Quickstart ```python -from intake import open_stac_catalog -catalog_url = 'https://raw.githubusercontent.com/cholmes/sample-stac/master/stac/catalog.json' -cat = open_stac_catalog(catalog_url) -cat['Houston-East-20170831-103f-100d-0f4f-RGB'].metadata -da = cat['Houston-East-20170831-103f-100d-0f4f-RGB']['thumbnail'].to_dask() +import intake + +catalog_url = 'https://www.planet.com/data/stac/catalog.json' +cat = intake.open_stac_catalog(catalog_url) + +collection = cat['planet-disaster-data'] +subset = collection['hurricane-harvey']['hurricane-harvey-0831'] +item = subset['Houston-East-20170831-103f-100d-0f4f-RGB'] + +da = item['thumbnail'].to_dask() da ``` The [examples/](examples/) directory contains several Jupyter Notebooks illustrating common workflows. +[STAC Index](https://stacindex.org/catalogs) is a convenient website for finding datasets with STACs + #### Versions To install a specific version of intake-stac, specify the version in the install command @@ -53,8 +60,7 @@ The table below shows the corresponding versions between intake-stac and STAC: | ----------- | ----------- | | 0.2.x | 0.6.x | | 0.3.x | 1.0.0-betaX | -| 0.4.x | 1.0.0-betaX | -| 1.0.0 | 1.0.0 | +| 0.4.x | 1.0.0 | ## About diff --git a/ci/environment-3.7.yml b/ci/environment-3.10.yml similarity index 91% rename from ci/environment-3.7.yml rename to ci/environment-3.10.yml index 4bbb9cb..95eebdb 100644 --- a/ci/environment-3.7.yml +++ b/ci/environment-3.10.yml @@ -2,7 +2,7 @@ name: intake-stac channels: - conda-forge dependencies: - - python=3.7 + - python=3.10 - fsspec - geopandas - intake diff --git a/ci/environment-dev.yml b/ci/environment-dev.yml index 8e42c20..755406b 100644 --- a/ci/environment-dev.yml +++ b/ci/environment-dev.yml @@ -1,9 +1,11 @@ -# everything except jupyterlab and plotting libraries +# Intake GUI also requires jupyterlab extensions installed +# conda env create -f ci/environment-dev.yml +# conda activate intake-stac-dev name: intake-stac-dev channels: - conda-forge dependencies: - - python=3.7 + - python - aiohttp - autopep8 - black @@ -11,9 +13,12 @@ dependencies: - codecov - coverage - dask + - datashader - distributed - flake8 - geopandas + - geoviews + - hvplot - intake - intake-geopandas - intake-parquet @@ -21,23 +26,28 @@ dependencies: - ipykernel - ipywidgets - isort + - jupyterlab - make + - matplotlib - nbsphinx - netcdf4 - numpy - numpydoc - pandoc + - panel - pip - pre_commit - pystac + - pystac-client + - pytest - pytest-cov - pytoml - pyyaml - rasterio - recommonmark - requests - - sat-search>=0.3 + - scikit-image - sphinx-copybutton - sphinx_rtd_theme - - sphinx>=1.6 + - sphinx >=1.6 - xarray diff --git a/ci/environment-docs.yml b/ci/environment-docs.yml index aeea6e4..a2dc083 100644 --- a/ci/environment-docs.yml +++ b/ci/environment-docs.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.9 + - python - aiohttp - fsspec - geopandas @@ -16,9 +16,9 @@ dependencies: - pillow - pip - pystac + - pystac-client - pytest-cov - rasterio - - sat-search - scikit-image - sphinx - sphinx_rtd_theme diff --git a/ci/environment-gui.yml b/ci/environment-gui.yml deleted file mode 100644 index e8119d3..0000000 --- a/ci/environment-gui.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Intake GUI also requires jupyterlab extensions installed -# conda env create -f ci/environment-gui-3.7.yml -# conda activate intake-stac-gui -# pip install -e . -# jupyter labextension install @pyviz/jupyterlab_pyviz -name: intake-stac-gui -channels: - - conda-forge -dependencies: - - python =3.7 - - aiohttp - - autopep8 - - black - - boto3 - - codecov - - coverage - - dask - - datashader - - distributed - - flake8 - - geopandas - - geoviews - - hvplot - - intake - - intake-geopandas - - intake-parquet - - intake-xarray - - ipykernel - - ipywidgets - - isort - - jupyterlab - - make - - matplotlib - - nbsphinx - - netcdf4 - - nodejs >=10.0.0 - - numpy - - numpydoc - - pandoc - - panel - - pip - - pre_commit - - pystac - - pytest - - pytest-cov - - pytest-icdiff - - pytoml - - pyyaml - - rasterio - - recommonmark - - requests - - sat-search>=0.3 - - scikit-image - - sphinx-copybutton - - sphinx_rtd_theme - - sphinx >=1.6 - - xarray diff --git a/ci/environment-upstream.yml b/ci/environment-upstream.yml index b870f67..aca5495 100644 --- a/ci/environment-upstream.yml +++ b/ci/environment-upstream.yml @@ -2,20 +2,15 @@ name: intake-stac channels: - conda-forge dependencies: - - python=3.8 - - fsspec + - python - geopandas - - intake - - intake-xarray - pip - - pystac - pytest-cov - rasterio - xarray - pip: - - git+https://github.com/intake/filesystem_spec.git + - git+https://github.com/fsspec/filesystem_spec.git - git+https://github.com/stac-utils/pystac.git - - git+https://github.com/sat-utils/sat-search.git - git+https://github.com/intake/intake.git - git+https://github.com/intake/intake-xarray.git - git+https://github.com/intake/intake_geopandas.git diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index dc58952..10a3b58 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -70,7 +70,7 @@ without using a local copy. This can be convenient for small fixes. .. code:: bash - $ conda env create -f ci/environment-dev.yml + $ conda env create -f ci/environment-docs.yml $ cd docs $ make html @@ -105,7 +105,7 @@ Preparing Pull Requests $ conda env create -f ci/environment-dev.yml $ conda activate intake-stac-dev - $ pip install . -e + $ pip install -e . #. Install `pre-commit `_ and its hook on the intake-stac repo:: diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 1d47d8c..2e6c128 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -11,8 +11,8 @@ Tutorial import pandas import xarray -Intake-stac simply provides a thin interface that combines `sat-stac` and -Intake. It's basic usage is shown below: +Intake-stac simply provides a thin interface that combines `pystac` and +`intake`. It's basic usage is shown below: To begin, import intake: @@ -24,44 +24,41 @@ Loading a catalog ----------------- You can load data from a STAC Catalog by providing the URL to valid STAC -Catalog: +Catalog (>1.0): .. ipython:: python - url = 'https://raw.githubusercontent.com/radiantearth/stac-spec/master/examples/catalog.json' + url = 'https://raw.githubusercontent.com/radiantearth/stac-spec/v1.0.0/examples/catalog.json' catalog = intake.open_stac_catalog(url) list(catalog) +Intake-Stac uses `pystac `_ to parse +STAC objects. You can also pass ``pystac`` objects (e.g. +``pystac.Catalog``) directly to the Intake-stac constructors: + +.. ipython:: python + + import pystac + root_url = 'https://raw.githubusercontent.com/relativeorbit/aws-rtc-12SYJ/main' + pystac_cat = pystac.read_file(f'{root_url}/catalog.json') + cat = intake.open_stac_catalog(pystac_cat) + You can also point to STAC Collections or Items. Each constructor returns a Intake Catalog with the top level corresponding to the STAC object used for initialization: .. ipython:: python - :verbatim: - root_url = 'https://raw.githubusercontent.com/sat-utils/sat-stac/master/test/catalog' stac_cat = intake.open_stac_catalog( f'{root_url}/catalog.json', ) collection_cat = intake.open_stac_collection( - f'{root_url}/eo/landsat-8-l1/catalog.json', + f'{root_url}/sentinel1-rtc-aws/collection.json', ) - items_cat = intake.open_stac_item( - f'{root_url}/eo/landsat-8-l1/item.json' + item_cat = intake.open_stac_item( + f'{root_url}/sentinel1-rtc-aws/12SYJ/2021/S1A_20210105_12SYJ_DSC/S1A_20210105_12SYJ_DSC.json' ) -Intake-Stac uses `pystac `_ to parse -STAC objects. You can also pass ``pystac`` objects (e.g. -``pystac.Catalog``) directly to the Intake-stac constructors: - -.. ipython:: python - :verbatim: - - import pystac - - pystac_cat = pystac.read_file(f'{root_url}/catalog.json') - cat = intake.open_stac_catalog(pystac_cat) - Using the catalog ----------------- @@ -70,25 +67,29 @@ contents: .. ipython:: python - print(list(catalog)) - cat = catalog['extensions-collection'] + print(list(stac_cat)) + cat = stac_cat['sentinel1-rtc-aws'] print(list(cat)) - subcat = cat['proj-example'] + subcat = cat['12SYJ'] - items = list(subcat) - print(items) + print(list(subcat)) + subsubcat = subcat['2021'] + print(list(subsubcat)[:3]) When you locate an item of interest, you have access to metadata and methods to load assets into Python objects .. ipython:: python - item = subcat['B1'] + item = subsubcat['S1A_20210105_12SYJ_DSC'] print(type(item)) print(item.metadata) + assets = list(item) + print(assets) + Loading a dataset ----------------- @@ -98,42 +99,37 @@ using Intake's `to_dask()` method. This reads only metadata, and streams values .. ipython:: python - da = item.to_dask() + da = item['gamma0_vv'].to_dask() display(da) -Working with `sat-search` -------------------------- +Working with `pystac-client` +---------------------------- -Intake-stac integrates with `sat-search` to faciliate dynamic search and +Intake-stac integrates with `pystac-client` to faciliate dynamic search and discovery of assets through a STAC-API. To begin, construct a search query -using `sat-search`: +using `pystac-client`: .. ipython:: python - import satsearch - print(satsearch.__version__) + import pystac_client + URL = "https://earth-search.aws.element84.com/v0" + catalog = pystac_client.Client.open(URL) - bbox = [35.48, -3.24, 35.58, -3.14] - dates = '2020-07-01/2020-08-15' - URL='https://earth-search.aws.element84.com/v0' - results = satsearch.Search.search(url=URL, - collections=['sentinel-s2-l2a-cogs'], - datetime=dates, - bbox=bbox, - sort=['-properties.datetime']) + results = catalog.search( + collections=["sentinel-s2-l2a-cogs"], + bbox = [35.48, -3.24, 35.58, -3.14], + datetime="2020-07-01/2020-08-15") - # 18 items found - items = results.items() + items = results.get_all_items() print(len(items)) - items.save('single-file-stac.json') -In the code section above, `items` is a `satstac.ItemsCollection` object. +In the code section above, `items` is a `pystac.ItemsCollection` object. Intake-stac can turn this object into an Intake catalog: .. ipython:: python - catalog = intake.open_stac_item_collection('single-file-stac.json') + catalog = intake.open_stac_item_collection(items) list(catalog) Using xarray-assets diff --git a/intake_stac/catalog.py b/intake_stac/catalog.py index a8bdfff..1faf220 100644 --- a/intake_stac/catalog.py +++ b/intake_stac/catalog.py @@ -347,7 +347,7 @@ def stack_bands(self, bands, path_as_pattern=None, concat_dim='band'): StacAsset with mapping of Asset names to Xarray bands Examples - ------- + -------- stack = item.stack_bands(['nir','red']) da = stack(chunks=dict(band=1, x=2048, y=2048)).to_dask() diff --git a/intake_stac/tests/test_catalog.py b/intake_stac/tests/test_catalog.py index 8c80d1f..bd3b8f8 100644 --- a/intake_stac/tests/test_catalog.py +++ b/intake_stac/tests/test_catalog.py @@ -290,7 +290,8 @@ def test_cat_item_yaml_roundtrip(self, pystac_item, tmp_path): class TestDrivers: def test_drivers_include_all_pystac_media_types(self): for media_type in pystac.MediaType: - assert media_type in drivers + if media_type != 'application/pdf': + assert media_type in drivers def test_drivers_can_open_all_earthsearch_sentinel_s2_l2a_cogs_assets(self): test_file = os.path.join(here, 'data/1.0.0beta2/earthsearch/single-file-stac.json') @@ -317,7 +318,7 @@ def test_cat_to_geopandas(pystac_itemcol): def test_collection_of_collection(): space = pystac.SpatialExtent([[0, 1, 2, 3]]) - time = pystac.TemporalExtent([datetime.datetime(2000, 1, 1), datetime.datetime(2000, 1, 1)]) + time = pystac.TemporalExtent([[datetime.datetime(2000, 1, 1), datetime.datetime(2000, 1, 1)]]) child = pystac.Collection('child', 'child-description', extent=pystac.Extent(space, time)) parent = pystac.Collection( 'parent', diff --git a/setup.py b/setup.py index 5963dc9..c08cdcf 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ LONG_DESCRIPTION = f.read() needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv) -PYTHON_REQUIRES = '>=3.6' +PYTHON_REQUIRES = '>=3.8' SETUP_REQUIRES = ['setuptools_scm'] if needs_pytest: SETUP_REQUIRES.appen('pytest-runner >= 4.2')