diff --git a/README.md b/README.md index fb69507..4a5a7dc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ The table below shows the corresponding versions between intake-stac and STAC: | 0.2.x | 0.6.x | | 0.3.x | 1.0.0-betaX | | 0.4.x | 1.0.0-betaX | +| 1.0.0 | 1.0.0 | ## About diff --git a/ci/environment-3.7.yml b/ci/environment-3.7.yml index 4bbb9cb..ae79196 100644 --- a/ci/environment-3.7.yml +++ b/ci/environment-3.7.yml @@ -7,7 +7,9 @@ dependencies: - geopandas - intake - intake-xarray - - pystac - pytest-cov - rasterio - xarray + - pip + - pip: + - pystac>=1.0.0rc1 diff --git a/ci/environment-3.8.yml b/ci/environment-3.8.yml index 4aab824..fa8914e 100644 --- a/ci/environment-3.8.yml +++ b/ci/environment-3.8.yml @@ -7,7 +7,8 @@ dependencies: - geopandas - intake - intake-xarray - - pystac - pytest-cov - rasterio - xarray + - pip: + - pystac>=1.0.0rc1 diff --git a/ci/environment-3.9.yml b/ci/environment-3.9.yml index 9bdcdff..0d38b60 100644 --- a/ci/environment-3.9.yml +++ b/ci/environment-3.9.yml @@ -7,7 +7,9 @@ dependencies: - geopandas - intake - intake-xarray - - pystac - pytest-cov - rasterio - xarray + - pip + - pip: + - pystac>=1.0.0rc1 diff --git a/ci/environment-unpinned.yml b/ci/environment-unpinned.yml index fb8680c..ab095d7 100644 --- a/ci/environment-unpinned.yml +++ b/ci/environment-unpinned.yml @@ -7,7 +7,9 @@ dependencies: - geopandas - intake - intake-xarray - - pystac - pytest-cov - rasterio - xarray + - pip + - pip: + - pystac>=1.0.0rc1 diff --git a/docs/source/api.rst b/docs/source/api.rst index a0325e9..367022b 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -29,4 +29,3 @@ Catalog Objects StacCollection StacItemCollection StacItem - catalog.StacEntry diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 2aaa649..b2acc0a 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -28,7 +28,7 @@ Catalog: .. ipython:: python - url = 'https://raw.githubusercontent.com/cholmes/sample-stac/master/stac/catalog.json' + url = 'https://raw.githubusercontent.com/radiantearth/stac-spec/master/examples/catalog.json' catalog = intake.open_stac_catalog(url) list(catalog) @@ -71,10 +71,10 @@ contents: .. ipython:: python print(list(catalog)) - cat = catalog['hurricane-harvey'] + cat = catalog['extensions-collection'] print(list(cat)) - subcat = cat['hurricane-harvey-0831'] + subcat = cat['proj-example'] items = list(subcat) print(items) @@ -85,29 +85,10 @@ When you locate an item of interest, you have access to metadata and methods to .. ipython:: python - item = subcat['Houston-East-20170831-103f-100d-0f4f-RGB'] + item = subcat['B1'] print(type(item)) print(item.metadata) - assets = list(item) - print(assets) - - asset = item['thumbnail'] - print(type(asset)) - print(asset.urlpath) - - -If the catalog has too many entries to comfortably print all at once, -you can narrow it by searching for a term (e.g. 'thumbnail'): - -.. ipython:: python - - for id, entry in subcat.search('thumbnail').items(): - print(id) - - asset = subcat['Houston-East-20170831-103f-100d-0f4f-RGB.thumbnail'] - print(asset.urlpath) - Loading a dataset ----------------- @@ -117,7 +98,7 @@ using Intake's `to_dask()` method. This reads only metadata, and streams values .. ipython:: python - da = asset.to_dask() + da = item.to_dask() display(da) diff --git a/intake_stac/catalog.py b/intake_stac/catalog.py index 8ed746f..e13f267 100644 --- a/intake_stac/catalog.py +++ b/intake_stac/catalog.py @@ -5,6 +5,7 @@ from intake.catalog import Catalog from intake.catalog.local import LocalCatalogEntry from pkg_resources import get_distribution +from pystac.extensions.eo import EOExtension __version__ = get_distribution('intake_stac').version @@ -173,21 +174,21 @@ class StacItemCollection(AbstractStacCatalog): """ name = 'stac_itemcollection' - _stac_cls = pystac.Catalog + _stac_cls = pystac.ItemCollection def _load(self): """ Load the STAC Item Collection. """ - if not self._stac_obj.ext.implements('single-file-stac'): - raise ValueError("StacItemCollection requires 'single-file-stac' extension") - for feature in self._stac_obj.ext['single-file-stac'].features: - self._entries[feature.id] = LocalCatalogEntry( - name=feature.id, + # if not self._stac_obj.ext.implements('single-file-stac'): + # raise ValueError("StacItemCollection requires 'single-file-stac' extension") + for item in self._stac_obj.items: + self._entries[item.id] = LocalCatalogEntry( + name=item.id, description='', driver=StacItem, catalog=self, - args={'stac_obj': feature}, + args={'stac_obj': item}, ) def to_geopandas(self, crs=None): @@ -246,25 +247,8 @@ def _get_band_info(self): Return list of band info dictionaries (name, common_name, etc.)... """ band_info = [] - try: - # NOTE: ensure we test these scenarios - # FileNotFoundError: [Errno 2] No such file or directory: '/catalog.json' - collection = self._stac_obj.get_collection() - if 'item-assets' in collection.stac_extensions: - for val in collection.ext['item_assets']: - if 'eo:bands' in val: - band_info.append(val.get('eo:bands')[0]) - else: - band_info = collection.summaries['eo:bands'] - - except Exception: - for band in self._stac_obj.ext['eo'].get_bands(): - band_info.append(band.to_dict()) - finally: - if not band_info: - raise ValueError( - 'Unable to parse "eo:bands" information from STAC Collection or Item Assets' - ) + for band in EOExtension.ext(self._stac_obj).bands: + band_info.append(band.to_dict()) return band_info def stack_bands(self, bands, path_as_pattern=None, concat_dim='band'): @@ -298,7 +282,7 @@ def stack_bands(self, bands, path_as_pattern=None, concat_dim='band'): stack = item.stack_bands(['B4','B5'], path_as_pattern='{band}.TIF') da = stack(chunks=dict(band=1, x=2048, y=2048)).to_dask() """ - if 'eo' not in self._stac_obj.stac_extensions: + if not EOExtension.has_extension(self._stac_obj): raise ValueError('STAC Item must implement "eo" extension to use this method') band_info = self._get_band_info() @@ -430,7 +414,9 @@ def _get_driver(self, asset): if entry_type in ['', 'null', None]: - suffix = os.path.splitext(asset.media_type)[-1] + suffix = '.tif' + if asset.media_type: + suffix = os.path.splitext(asset.media_type)[-1] if suffix in ['.nc', '.h5', '.hdf']: asset.media_type = 'application/netcdf' warnings.warn( diff --git a/intake_stac/tests/data/1.0.0/catalog/catalog.json b/intake_stac/tests/data/1.0.0/catalog/catalog.json new file mode 100644 index 0000000..799c593 --- /dev/null +++ b/intake_stac/tests/data/1.0.0/catalog/catalog.json @@ -0,0 +1,19 @@ +{ + "type": "Catalog", + "id": "test", + "stac_version": "1.0.0", + "description": "test catalog", + "links": [ + { + "rel": "child", + "href": "./child-catalog.json", + "type": "application/json" + }, + { + "rel": "root", + "href": "./catalog.json", + "type": "application/json" + } + ], + "stac_extensions": [] +} diff --git a/intake_stac/tests/data/1.0.0/catalog/child-catalog.json b/intake_stac/tests/data/1.0.0/catalog/child-catalog.json new file mode 100644 index 0000000..df5db51 --- /dev/null +++ b/intake_stac/tests/data/1.0.0/catalog/child-catalog.json @@ -0,0 +1,14 @@ +{ + "type": "Catalog", + "id": "test", + "stac_version": "1.0.0", + "description": "child catalog", + "links": [ + { + "rel": "root", + "href": "./catalog.json", + "type": "application/json" + } + ], + "stac_extensions": [] +} diff --git a/intake_stac/tests/data/1.0.0/collection/collection.json b/intake_stac/tests/data/1.0.0/collection/collection.json new file mode 100644 index 0000000..96bbd39 --- /dev/null +++ b/intake_stac/tests/data/1.0.0/collection/collection.json @@ -0,0 +1,75 @@ +{ + "id": "simple-collection", + "type": "Collection", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json" + ], + "stac_version": "1.0.0", + "description": "A simple collection demonstrating core catalog fields with links to a couple of items", + "title": "Simple Example Collection", + "providers": [ + { + "name": "Remote Data, Inc", + "description": "Producers of awesome spatiotemporal assets", + "roles": ["producer", "processor"], + "url": "http://remotedata.io" + } + ], + "extent": { + "spatial": { + "bbox": [ + [ + 172.91173669923782, 1.3438851951615003, 172.95469614953714, + 1.3690476620161975 + ] + ] + }, + "temporal": { + "interval": [["2020-12-11T22:38:32.125Z", "2020-12-14T18:02:31.437Z"]] + } + }, + "license": "CC-BY-4.0", + "summaries": { + "platform": ["cool_sat1", "cool_sat2"], + "constellation": ["ion"], + "instruments": ["cool_sensor_v1", "cool_sensor_v2"], + "gsd": { + "minimum": 0.512, + "maximum": 0.66 + }, + "eo:cloud_cover": { + "minimum": 1.2, + "maximum": 1.2 + }, + "proj:epsg": { + "minimum": 32659, + "maximum": 32659 + }, + "view:sun_elevation": { + "minimum": 54.9, + "maximum": 54.9 + }, + "view:off_nadir": { + "minimum": 3.8, + "maximum": 3.8 + }, + "view:sun_azimuth": { + "minimum": 135.7, + "maximum": 135.7 + } + }, + "links": [ + { + "rel": "root", + "href": "./collection.json", + "type": "application/json" + }, + { + "rel": "item", + "href": "./simple-item.json", + "type": "application/geo+json", + "title": "Simple Item" + } + ] +} diff --git a/intake_stac/tests/data/1.0.0/collection/simple-item.json b/intake_stac/tests/data/1.0.0/collection/simple-item.json new file mode 100644 index 0000000..0c8547f --- /dev/null +++ b/intake_stac/tests/data/1.0.0/collection/simple-item.json @@ -0,0 +1,87 @@ +{ + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json" + ], + "type": "Feature", + "id": "S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101", + "bbox": [ + 172.91173669923782, 1.3438851951615003, 172.95469614953714, + 1.3690476620161975 + ], + "geometry": { + "coordinates": [ + [ + [-82.89978, 18.98277161], + [-81.85693, 18.99053787], + [-81.85202, 17.99825755], + [-82.888855, 17.99092482], + [-82.89978, 18.98277161] + ] + ], + "type": "Polygon" + }, + "properties": { + "datetime": "2017-12-27T16:04:59.027000Z" + }, + "collection": "simple-collection", + "links": [ + { + "rel": "collection", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection" + }, + { + "rel": "root", + "href": "./collection.json", + "type": "application/json" + }, + { + "rel": "parent", + "href": "./collection.json", + "type": "application/json" + } + ], + "assets": { + "B02": { + "href": "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/17/Q/LA/2017/12/27/S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101.SAFE/GRANULE/L2A_T17QLA_A004227_20171227T160750/IMG_DATA/R10m/T17QLA_20171227T160459_B02_10m.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Band 2 - Blue", + "eo:bands": [ + { + "name": "B02", + "common_name": "blue", + "description": "Band 2 - Blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:bbox": [300000, 1990200, 409800, 2100000], + "proj:transform": [10, 0, 300000, 0, -10, 2100000], + "roles": ["data"] + }, + "B03": { + "href": "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/17/Q/LA/2017/12/27/S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101.SAFE/GRANULE/L2A_T17QLA_A004227_20171227T160750/IMG_DATA/R10m/T17QLA_20171227T160459_B03_10m.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Band 3 - Green", + "eo:bands": [ + { + "name": "B03", + "common_name": "green", + "description": "Band 3 - Green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + } + ], + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:bbox": [300000, 1990200, 409800, 2100000], + "proj:transform": [10, 0, 300000, 0, -10, 2100000], + "roles": ["data"] + } + } +} diff --git a/intake_stac/tests/data/1.0.0/itemcollection/example-search.json b/intake_stac/tests/data/1.0.0/itemcollection/example-search.json new file mode 100644 index 0000000..3602107 --- /dev/null +++ b/intake_stac/tests/data/1.0.0/itemcollection/example-search.json @@ -0,0 +1,350 @@ +{ + "id": "mysearchresults", + "stac_version": "1.0.0-beta.2", + "stac_extensions": ["single-file-stac"], + "description": "A bunch of results from a search", + "type": "FeatureCollection", + "features": [ + { + "stac_version": "1.0.0-beta.2", + "stac_extensions": [ + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json" + ], + "type": "Feature", + "id": "LC80370332018039LGN00", + "collection": "landsat-8-l1", + "bbox": [-112.21054, 37.83042, -109.4992, 39.95532], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-111.6768167850251, 39.952817693022276], + [-109.5010938553632, 39.55607811527241], + [-110.03573868784865, 37.83172334507642], + [-112.20846353249907, 38.236456540046845], + [-111.6768167850251, 39.952817693022276] + ] + ] + }, + "properties": { + "datetime": "2018-02-08T18:02:15.719478+00:00", + "view:sun_azimuth": 152.63804142, + "view:sun_elevation": 31.82216637, + "proj:epsg": 32612 + }, + "assets": { + "index": { + "type": "text/html", + "title": "HTML index page", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/index.html" + }, + "thumbnail": { + "title": "Thumbnail image", + "type": "image/jpeg", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_thumb_large.jpg" + }, + "B1": { + "type": "image/tiff; application=geotiff", + "title": "Band 1 (coastal)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B1.TIF" + }, + "B2": { + "type": "image/tiff; application=geotiff", + "title": "Band 2 (blue)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B2.TIF" + }, + "B3": { + "type": "image/tiff; application=geotiff", + "title": "Band 3 (green)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B3.TIF" + }, + "B4": { + "type": "image/tiff; application=geotiff", + "title": "Band 4 (red)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B4.TIF" + }, + "B5": { + "type": "image/tiff; application=geotiff", + "title": "Band 5 (nir)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B5.TIF" + }, + "B6": { + "type": "image/tiff; application=geotiff", + "title": "Band 6 (swir16)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B6.TIF" + }, + "B7": { + "type": "image/tiff; application=geotiff", + "title": "Band 7 (swir22)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B7.TIF" + }, + "B8": { + "type": "image/tiff; application=geotiff", + "title": "Band 8 (pan)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B8.TIF" + }, + "B9": { + "type": "image/tiff; application=geotiff", + "title": "Band 9 (cirrus)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B9.TIF" + }, + "B10": { + "type": "image/tiff; application=geotiff", + "title": "Band 10 (lwir)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B10.TIF" + }, + "B11": { + "type": "image/tiff; application=geotiff", + "title": "Band 11 (lwir)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B11.TIF" + }, + "ANG": { + "title": "Angle coefficients file", + "type": "text/plain", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_ANG.txt" + }, + "MTL": { + "title": "original metadata file", + "type": "text/plain", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_MTL.txt" + }, + "BQA": { + "title": "Band quality data", + "type": "image/tiff; application=geotiff", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_BQA.TIF" + } + }, + "links": [] + }, + { + "stac_version": "1.0.0", + "stac_extensions": ["projection", "view"], + "type": "Feature", + "id": "LC80340332018034LGN00", + "collection": "landsat-8-l1", + "bbox": [-107.6044, 37.8096, -104.86884, 39.97508], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-107.03912158283073, 39.975078807631036], + [-104.87161559271382, 39.548160703908025], + [-105.43927721248009, 37.81075859503169], + [-107.60423259994965, 38.24485405534073], + [-107.03912158283073, 39.975078807631036] + ] + ] + }, + "properties": { + "datetime": "2018-02-03T17:43:44Z", + "view:sun_azimuth": 153.39513457, + "view:sun_elevation": 30.41894816, + "proj:epsg": 32613 + }, + "assets": { + "index": { + "type": "text/html", + "title": "HTML index page", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/index.html" + }, + "thumbnail": { + "title": "Thumbnail image", + "type": "image/jpeg", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_thumb_large.jpg" + }, + "B1": { + "type": "image/tiff; application=geotiff", + "title": "Band 1 (coastal)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B1.TIF" + }, + "B2": { + "type": "image/tiff; application=geotiff", + "title": "Band 2 (blue)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B2.TIF" + }, + "B3": { + "type": "image/tiff; application=geotiff", + "title": "Band 3 (green)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B3.TIF" + }, + "B4": { + "type": "image/tiff; application=geotiff", + "title": "Band 4 (red)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B4.TIF" + }, + "B5": { + "type": "image/tiff; application=geotiff", + "title": "Band 5 (nir)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B5.TIF" + }, + "B6": { + "type": "image/tiff; application=geotiff", + "title": "Band 6 (swir16)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B6.TIF" + }, + "B7": { + "type": "image/tiff; application=geotiff", + "title": "Band 7 (swir22)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B7.TIF" + }, + "B8": { + "type": "image/tiff; application=geotiff", + "title": "Band 8 (pan)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B8.TIF" + }, + "B9": { + "type": "image/tiff; application=geotiff", + "title": "Band 9 (cirrus)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B9.TIF" + }, + "B10": { + "type": "image/tiff; application=geotiff", + "title": "Band 10 (lwir)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B10.TIF" + }, + "B11": { + "type": "image/tiff; application=geotiff", + "title": "Band 11 (lwir)", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B11.TIF" + }, + "ANG": { + "title": "Angle coefficients file", + "type": "text/plain", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_ANG.txt" + }, + "MTL": { + "title": "original metadata file", + "type": "text/plain", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_MTL.txt" + }, + "BQA": { + "title": "Band quality data", + "type": "image/tiff; application=geotiff", + "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_BQA.TIF" + } + }, + "links": [] + } + ], + "collections": [ + { + "id": "landsat-8-l1", + "title": "Landsat 8 L1", + "description": "Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.", + "keywords": ["landsat", "earth observation", "usgs"], + "stac_version": "1.0.0-beta.2", + "stac_extensions": ["item_assets"], + "extent": { + "spatial": { + "bbox": [[-180, -90, 180, 90]] + }, + "temporal": { + "interval": [["2013-06-01T00:00:00Z", null]] + } + }, + "providers": [ + { + "name": "USGS", + "roles": ["producer"], + "url": "https://landsat.usgs.gov/" + }, + { + "name": "Planet Labs", + "roles": ["processor"], + "url": "https://github.com/landsat-pds/landsat_ingestor" + }, + { + "name": "AWS", + "roles": ["host"], + "url": "https://landsatonaws.com/" + }, + { + "name": "Development Seed", + "roles": ["processor"], + "url": "https://github.com/sat-utils/sat-api" + } + ], + "license": "PDDL-1.0", + "summaries": { + "gsd": [15], + "platform": ["landsat-8"], + "instruments": ["oli", "tirs"], + "view:off_nadir": [0] + }, + "item_assets": { + "index": { + "type": "text/html", + "title": "HTML index page" + }, + "thumbnail": { + "title": "Thumbnail image", + "type": "image/jpeg" + }, + "B1": { + "type": "image/tiff; application=geotiff", + "title": "Band 1 (coastal)" + }, + "B2": { + "type": "image/tiff; application=geotiff", + "title": "Band 2 (blue)" + }, + "B3": { + "type": "image/tiff; application=geotiff", + "title": "Band 3 (green)" + }, + "B4": { + "type": "image/tiff; application=geotiff", + "title": "Band 4 (red)" + }, + "B5": { + "type": "image/tiff; application=geotiff", + "title": "Band 5 (nir)" + }, + "B6": { + "type": "image/tiff; application=geotiff", + "title": "Band 6 (swir16)" + }, + "B7": { + "type": "image/tiff; application=geotiff", + "title": "Band 7 (swir22)" + }, + "B8": { + "type": "image/tiff; application=geotiff", + "title": "Band 8 (pan)" + }, + "B9": { + "type": "image/tiff; application=geotiff", + "title": "Band 9 (cirrus)" + }, + "B10": { + "type": "image/tiff; application=geotiff", + "title": "Band 10 (lwir)" + }, + "B11": { + "type": "image/tiff; application=geotiff", + "title": "Band 11 (lwir)" + }, + "ANG": { + "title": "Angle coefficients file", + "type": "text/plain" + }, + "MTL": { + "title": "original metadata file", + "type": "text/plain" + }, + "BQA": { + "title": "Band quality data", + "type": "image/tiff; application=geotiff" + } + }, + "links": [ + { + "rel": "self", + "href": "./example-search.json" + } + ] + } + ], + "links": [] +} diff --git a/intake_stac/tests/test_catalog.py b/intake_stac/tests/test_catalog.py index e601612..d4fc659 100644 --- a/intake_stac/tests/test_catalog.py +++ b/intake_stac/tests/test_catalog.py @@ -1,6 +1,7 @@ import datetime import os.path import sys +from pathlib import Path import intake import pystac @@ -11,21 +12,13 @@ from intake_stac import StacCatalog, StacCollection, StacItem, StacItemCollection from intake_stac.catalog import CombinedAssets, StacAsset -here = os.path.dirname(__file__) +here = Path(__file__).parent -# sat-stac examples -# ----- -sat_stac_repo = 'https://raw.githubusercontent.com/sat-utils/sat-stac/master' -cat_url = f'{sat_stac_repo}/test/catalog/catalog.json' -col_url = f'{sat_stac_repo}/test/catalog/eo/sentinel-2-l1c/catalog.json' -item_url = f'{sat_stac_repo}/test/catalog/eo/landsat-8-l1/item.json' -# pystac examples -# ----- -pystac_repo = 'https://raw.githubusercontent.com/stac-utils/pystac/develop/tests/data-files' -itemcol_url = ( - f'{pystac_repo}/examples/1.0.0-beta.2/extensions/single-file-stac/examples/example-search.json' -) +cat_url = str(here / 'data/1.0.0/catalog/catalog.json') +col_url = str(here / 'data/1.0.0/collection/collection.json') +item_url = str(here / 'data/1.0.0/collection/simple-item.json') +itemcol_url = str(here / 'data/1.0.0/itemcollection/example-search.json') @pytest.fixture(scope='module') @@ -35,7 +28,8 @@ def pystac_cat(): @pytest.fixture(scope='module') def pystac_col(): - return pystac.Collection.from_file(col_url) + col = pystac.Collection.from_file(col_url) + return col @pytest.fixture(scope='module') @@ -45,7 +39,9 @@ def pystac_item(): @pytest.fixture(scope='module') def pystac_itemcol(): - return pystac.read_file(itemcol_url) + # return pystac.read_file(itemcol_url) + # ItemCollection is not a valid pystac STACObject, so can't use read_file. + return pystac.ItemCollection.from_file(itemcol_url) @pytest.fixture(scope='module') @@ -53,184 +49,197 @@ def intake_stac_cat(): return StacCatalog.from_url(cat_url) -def test_init_catalog_from_url(): - cat = StacCatalog(cat_url) - assert isinstance(cat, intake.catalog.Catalog) - assert cat.name == 'stac-catalog' - assert cat.discover()['container'] == 'catalog' - assert int(cat.metadata['stac_version'][0]) >= 1 - - cat = StacCatalog.from_url(cat_url) - assert isinstance(cat, intake.catalog.Catalog) - assert cat.name == 'stac-catalog' - assert cat.discover()['container'] == 'catalog' - assert int(cat.metadata['stac_version'][0]) >= 1 - - # test kwargs are passed through - cat = StacCatalog.from_url(cat_url, name='intake-stac-test') - assert 'intake-stac-test' == cat.name - - -def test_init_catalog_from_pystac_obj(pystac_cat): - cat = StacCatalog(pystac_cat) - assert isinstance(cat, intake.catalog.Catalog) - assert cat.discover()['container'] == 'catalog' - assert cat.name == 'stac-catalog' - assert cat.name == pystac_cat.id - - # test kwargs are passed through - cat = StacCatalog(pystac_cat, name='intake-stac-test') - assert 'intake-stac-test' == cat.name - - -def test_init_catalog_with_wrong_type_raises(pystac_cat): - with pytest.raises(ValueError): - StacCollection(pystac_cat) - - -def test_init_catalog_with_bad_url_raises(): - # json.decoder.JSONDecodeError or FileNotFoundError - with pytest.raises(Exception): - StacCatalog('https://raw.githubusercontent.com/') - - -def test_serialize(intake_stac_cat): - cat_str = intake_stac_cat.serialize() - assert isinstance(cat_str, str) - - -def test_cat_entries(intake_stac_cat): - assert list(intake_stac_cat) - assert all([isinstance(v, (LocalCatalogEntry, Catalog)) for _, v in intake_stac_cat.items()]) - - -def test_cat_name_from_pystac_catalog_id(intake_stac_cat): - assert intake_stac_cat.name == 'stac-catalog' - - -def test_cat_from_collection(pystac_col): - cat = StacCollection(pystac_col) - subcat_name = 'sentinel-2a-catalog' - item_name = 'S2B_25WFU_20200610_0_L1C' - assert cat.name == pystac_col.id - assert subcat_name in cat - assert item_name in cat[subcat_name] - assert 'B04' in cat[subcat_name][item_name] - - -def test_cat_from_item_collection(pystac_itemcol): - cat = StacItemCollection(pystac_itemcol) - assert 'LC80340332018034LGN00' in cat - assert 'B5' in cat.LC80340332018034LGN00 - - -def test_cat_from_item(pystac_item): - cat = StacItem(pystac_item) - assert 'B5' in cat - - -def test_cat_item_stacking(pystac_item): - item = StacItem(pystac_item) - list_of_bands = ['B1', 'B2'] - new_entry = item.stack_bands(list_of_bands) - assert isinstance(new_entry, CombinedAssets) - assert new_entry._description == 'B1, B2' - assert new_entry.name == 'B1_B2' - new_da = new_entry().to_dask() - assert sorted([dim for dim in new_da.dims]) == ['band', 'x', 'y'] - - -def test_cat_item_stacking_using_common_name(pystac_item): - item = StacItem(pystac_item) - list_of_bands = ['coastal', 'blue'] - new_entry = item.stack_bands(list_of_bands) - assert isinstance(new_entry, CombinedAssets) - assert new_entry._description == 'B1, B2' - assert new_entry.name == 'coastal_blue' - new_da = new_entry().to_dask() - assert sorted([dim for dim in new_da.dims]) == ['band', 'x', 'y'] - - -def test_cat_item_stacking_path_as_pattern(pystac_item): - item = StacItem(pystac_item) - list_of_bands = ['B1', 'B2'] - new_entry = item.stack_bands(list_of_bands, path_as_pattern='{}{band:2}.TIF') - assert isinstance(new_entry, CombinedAssets) - new_da = new_entry().to_dask() - assert (new_da.band == ['B1', 'B2']).all() - - -def test_cat_item_stacking_dims_of_different_type_raises_error(pystac_item): - item = StacItem(pystac_item) - list_of_bands = ['B1', 'ANG'] - with pytest.raises(ValueError, match=('ANG not found in list of eo:bands in collection')): - item.stack_bands(list_of_bands) - - -def test_cat_item_stacking_dims_with_nonexistent_band_raises_error(pystac_item,): # noqa: E501 - item = StacItem(pystac_item) - list_of_bands = ['B1', 'foo'] - with pytest.raises(ValueError, match="'B8', 'B9', 'blue', 'cirrus'"): - item.stack_bands(list_of_bands) - - -def test_cat_item_stacking_dims_of_different_size_regrids(pystac_item): - item = StacItem(pystac_item) - list_of_bands = ['B1', 'B8'] - B1_da = item.B1.to_dask() - assert B1_da.shape == (1, 7791, 7651) - B8_da = item.B8.to_dask() - assert B8_da.shape == (1, 15581, 15301) - new_entry = item.stack_bands(list_of_bands) - new_da = new_entry().to_dask() - assert new_da.shape == (2, 15581, 15301) - assert sorted([dim for dim in new_da.dims]) == ['band', 'x', 'y'] - - -def test_asset_describe(pystac_item): - item = StacItem(pystac_item) - key = 'B1' - asset = item[key] - d = asset.describe() - - assert d['name'] == key - assert d['container'] == 'xarray' - assert d['plugin'] == ['rasterio'] - assert d['args']['urlpath'] == asset.urlpath - assert d['description'] == asset.description - # NOTE: note sure why asset.metadata has 'catalog_dir' key ? - # assert d['metadata'] == asset.metadata - - -def test_asset_missing_type(pystac_item): - key = 'B1' - asset = pystac_item.assets.get('B1') - asset.media_type = '' - with pytest.warns(Warning, match='STAC Asset'): +class TestCatalog: + def test_init_catalog_from_url(self): + cat = StacCatalog(cat_url) + assert isinstance(cat, intake.catalog.Catalog) + assert cat.name == 'test' + assert cat.discover()['container'] == 'catalog' + assert int(cat.metadata['stac_version'][0]) >= 1 + + cat = StacCatalog.from_url(cat_url) + assert isinstance(cat, intake.catalog.Catalog) + assert cat.name == 'test' + assert cat.discover()['container'] == 'catalog' + assert int(cat.metadata['stac_version'][0]) >= 1 + + # test kwargs are passed through + cat = StacCatalog.from_url(cat_url, name='intake-stac-test') + assert 'intake-stac-test' == cat.name + + def test_init_catalog_from_pystac_obj(self, pystac_cat): + cat = StacCatalog(pystac_cat) + assert isinstance(cat, intake.catalog.Catalog) + assert cat.discover()['container'] == 'catalog' + assert cat.name == 'test' + assert cat.name == pystac_cat.id + + # test kwargs are passed through + cat = StacCatalog(pystac_cat, name='intake-stac-test') + assert 'intake-stac-test' == cat.name + + def test_init_catalog_with_wrong_type_raises(self, pystac_cat): + with pytest.raises(ValueError): + StacCollection(pystac_cat) + + def test_init_catalog_with_bad_url_raises(self): + # json.decoder.JSONDecodeError or FileNotFoundError + with pytest.raises(Exception): + StacCatalog('https://raw.githubusercontent.com/') + + def test_serialize(self, intake_stac_cat): + cat_str = intake_stac_cat.serialize() + assert isinstance(cat_str, str) + + def test_cat_entries(self, intake_stac_cat): + assert list(intake_stac_cat) + assert all( + [isinstance(v, (LocalCatalogEntry, Catalog)) for _, v in intake_stac_cat.items()] + ) + + def test_cat_name_from_pystac_catalog_id(self, intake_stac_cat): + assert intake_stac_cat.name == 'test' + + +class TestCollection: + def test_cat_from_collection(self, pystac_col): + cat = StacCollection(pystac_col) + subcat_name = 'S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101' + assert cat.name == pystac_col.id + assert subcat_name in cat + # This is taking way too long + # item_name = 'S2B_25WFU_20200610_0_L1C' + # assert item_name in cat[subcat_name] + # assert 'B04' in cat[subcat_name][item_name] + + +class TestItemCollection: + def test_cat_from_item_collection(self, pystac_itemcol): + cat = StacItemCollection(pystac_itemcol) + assert 'LC80340332018034LGN00' in cat + assert 'B5' in cat.LC80340332018034LGN00 + + @pytest.mark.parametrize('crs', ['IGNF:ETRS89UTM28', 'epsg:26909']) + def test_cat_to_geopandas_crs(self, crs, pystac_itemcol): + nfeatures = len(pystac_itemcol.items) + geopandas = pytest.importorskip('geopandas') + + cat = StacItemCollection(pystac_itemcol) + df = cat.to_geopandas(crs=crs) + assert isinstance(df, geopandas.GeoDataFrame) + assert len(df) == nfeatures + assert df.crs == crs + + def test_cat_to_missing_geopandas(self, pystac_itemcol, monkeypatch): + from unittest import mock + + with pytest.raises(ImportError): + with mock.patch.dict(sys.modules, {'geopandas': None}): + cat = StacItemCollection(pystac_itemcol) + _ = cat.to_geopandas() + + def test_load_satsearch_results(self, pystac_itemcol): + test_file = os.path.join(here, 'data/1.0.0beta2/earthsearch/single-file-stac.json') + catalog = intake.open_stac_item_collection(test_file) + assert isinstance(catalog, StacItemCollection) + assert len(catalog) == 18 + + +class TestItem: + def test_cat_from_item(self, pystac_item): + cat = StacItem(pystac_item) + assert 'B02' in cat + + def test_cat_item_stacking(self, pystac_item): + item = StacItem(pystac_item) + list_of_bands = ['B02', 'B03'] + new_entry = item.stack_bands(list_of_bands) + assert isinstance(new_entry, CombinedAssets) + assert new_entry._description == 'B02, B03' + assert new_entry.name == 'B02_B03' + + def test_cat_item_stacking_common_name(self, pystac_item): + item = StacItem(pystac_item) + list_of_bands = ['blue', 'green'] + new_entry = item.stack_bands(list_of_bands) + assert isinstance(new_entry, CombinedAssets) + assert new_entry._description == 'B02, B03' + assert new_entry.name == 'blue_green' + + def test_cat_item_stacking_path_as_pattern(self, pystac_item): + item = StacItem(pystac_item) + list_of_bands = ['B02', 'B03'] + new_entry = item.stack_bands(list_of_bands, path_as_pattern='{}{band:2}.TIF') + assert isinstance(new_entry, CombinedAssets) + + def test_cat_item_stacking_dims_of_different_type_raises_error(self, pystac_item): + item = StacItem(pystac_item) + list_of_bands = ['B02', 'ANG'] + with pytest.raises(ValueError, match=('ANG not found in list of eo:bands in collection')): + item.stack_bands(list_of_bands) + + def test_cat_item_stacking_dims_with_nonexistent_band_raises_error( + self, pystac_item, + ): # noqa: E501 + item = StacItem(pystac_item) + list_of_bands = ['B01', 'foo'] + with pytest.raises(ValueError, match="'B02', 'B03', 'blue', 'green'"): + item.stack_bands(list_of_bands) + + # def test_cat_item_stacking_dims_of_different_size_regrids(self, pystac_item): + # item = StacItem(pystac_item) + # list_of_bands = ['B1', 'B8'] + # B1_da = item.B1.to_dask() + # assert B1_da.shape == (1, 8391, 8311) + # B8_da = item.B8.to_dask() + # assert B8_da.shape == (1, 16781, 16621) + # new_entry = item.stack_bands(list_of_bands) + # new_da = new_entry().to_dask() + # assert new_da.shape == (2, 16781, 16621) + # assert sorted([dim for dim in new_da.dims]) == ['band', 'x', 'y'] + + def test_asset_describe(self, pystac_item): + item = StacItem(pystac_item) + key = 'B02' + asset = item[key] + d = asset.describe() + + assert d['name'] == key + assert d['container'] == 'xarray' + assert d['plugin'] == ['rasterio'] + assert d['args']['urlpath'] == asset.urlpath + assert d['description'] == asset.description + # NOTE: note sure why asset.metadata has 'catalog_dir' key ? + # assert d['metadata'] == asset.metadata + + def test_asset_missing_type(self, pystac_item): + key = 'B02' + asset = pystac_item.assets.get('B02') + asset.media_type = '' + with pytest.warns(Warning, match='STAC Asset'): + entry = StacAsset(key, asset) + d = entry.describe() + + assert d['name'] == key + assert d['metadata']['type'] == 'application/rasterio' # default_type + assert d['container'] == 'xarray' + assert d['plugin'] == ['rasterio'] + + def test_asset_unknown_type(self, pystac_item): + key = 'B02' + asset = pystac_item.assets.get('B02') + asset.media_type = 'unrecognized' entry = StacAsset(key, asset) - d = entry.describe() - - assert d['name'] == key - assert d['metadata']['type'] == 'application/rasterio' # default_type - assert d['container'] == 'xarray' - assert d['plugin'] == ['rasterio'] - - -def test_asset_unknown_type(pystac_item): - key = 'B1' - asset = pystac_item.assets.get('B1') - asset.media_type = 'unrecognized' - entry = StacAsset(key, asset) - d = entry.describe() + d = entry.describe() - assert d['name'] == key - assert d['metadata']['type'] == 'unrecognized' - assert d['container'] == 'xarray' - assert d['plugin'] == ['rasterio'] + assert d['name'] == key + assert d['metadata']['type'] == 'unrecognized' + assert d['container'] == 'xarray' + assert d['plugin'] == ['rasterio'] def test_cat_to_geopandas(pystac_itemcol): - nfeatures = len(pystac_itemcol.ext['single-file-stac'].features) + nfeatures = len(pystac_itemcol) geopandas = pytest.importorskip('geopandas') cat = StacItemCollection(pystac_itemcol) @@ -244,34 +253,6 @@ def test_cat_to_geopandas(pystac_itemcol): assert epsg == 4326 -@pytest.mark.parametrize('crs', ['IGNF:ETRS89UTM28', 'epsg:26909']) -def test_cat_to_geopandas_crs(crs, pystac_itemcol): - nfeatures = len(pystac_itemcol.ext['single-file-stac'].features) - geopandas = pytest.importorskip('geopandas') - - cat = StacItemCollection(pystac_itemcol) - df = cat.to_geopandas(crs=crs) - assert isinstance(df, geopandas.GeoDataFrame) - assert len(df) == nfeatures - assert df.crs == crs - - -def test_cat_to_missing_geopandas(pystac_itemcol, monkeypatch): - from unittest import mock - - with pytest.raises(ImportError): - with mock.patch.dict(sys.modules, {'geopandas': None}): - cat = StacItemCollection(pystac_itemcol) - _ = cat.to_geopandas() - - -def test_load_satsearch_results(pystac_itemcol): - test_file = os.path.join(here, 'data/1.0.0beta2/earthsearch/single-file-stac.json') - catalog = intake.open_stac_item_collection(test_file) - assert isinstance(catalog, StacItemCollection) - assert len(catalog) == 18 - - def test_collection_of_collection(): space = pystac.SpatialExtent([[0, 1, 2, 3]]) time = pystac.TemporalExtent([datetime.datetime(2000, 1, 1), datetime.datetime(2000, 1, 1)]) diff --git a/requirements.txt b/requirements.txt index 8618ee7..4999632 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ fsspec>=0.8.4 intake>=0.5.1 intake-xarray>=0.4 -pystac>=0.5.* +pystac>=1.0.0rc1