Merge branch '470-url-based-s3-asset' into 'dev'

Resolve "Add AWS landsat fetch"

Closes #470 and #480

See merge request appliedgeosolutions/gips!490

README.md

@@ -36,6 +36,37 @@ https://lpdaac.usgs.gov/about/news_archive/important_changes_lp_daac_data_access
 
 For information on setting up automated testing, see `gips/test/README.md`.
 
+### Landsat S3 Configuration Note
+
+GIPS supports Landsat data access via AWS S3:
+
+https://aws.amazon.com/public-datasets/landsat/
+
+Most GIPS products are supported if you have API access to AWS:
+
+https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/
+
+Provide the special access tokens via environment variable:
+
+```
+export AWS_ACCESS_KEY_ID='your-id'
+export AWS_SECRET_ACCESS_KEY='your-key'
+```
+
+Finally set `settings.py` to tell gips to fetch C1 assets from S3:
+
+```
+REPOS = {
+    'landsat': {
+        # . . .
+        'source': 's3', # default is 'usgs'
+    }
+}
+```
+
+After this is done, `gips_inventory --fetch`, `gips_inventory`, and
+`gips_process -p <product-list-here>` should work for S3 assets.
+
 ## Environment
 
 `GIPS_ORM` controls whether or not the GIPS ORM is activated.  The ORM is

gips/data/core.py

@@ -67,6 +67,8 @@ class Repository(object):
     # valid sub directories in repo
     _subdirs = ['tiles', 'stage', 'quarantine', 'composites']
 
+    default_settings = {}
+
     @classmethod
     def feature2tile(cls, feature):
         """ Get tile designation from a geospatial feature (i.e. a row) """
@@ -110,21 +112,28 @@ def find_dates(cls, tile):
     ##########################################################################
     @classmethod
     def get_setting(cls, key):
-        """ Get value from repo settings """
-        dataclass = cls.__name__[:-10]
+        """Get given setting from settings.REPOS[driver].
+
+        If the key isn't found, it attempts to load a default from
+        cls.default_settings, a dict of such things.  If still not found,
+        resorts to magic for 'driver' and 'tiles', ValueError otherwise.
+        """
+        dataclass = cls.__name__[:-10] # name of a class, not the class object
         r = settings().REPOS[dataclass]
-        if key not in r.keys():
-            # not in settings file, use defaults
-            exec('import gips.data.%s as clsname' % dataclass)
-            driverpath = os.path.dirname(clsname.__file__)
-            if key == 'driver':
-                return driverpath
-            elif key == 'tiles':
-                return os.path.join(driverpath, 'tiles.shp')
-            else:
-                raise Exception('%s is not a valid setting!' % key)
-        else:
+        if key in r:
             return r[key]
+        if key in cls.default_settings:
+            return cls.default_settings[key]
+
+        # not in settings file nor default, so resort to magic
+        exec('import gips.data.%s as clsname' % dataclass)
+        driverpath = os.path.dirname(clsname.__file__)
+        if key == 'driver':
+            return driverpath
+        if key == 'tiles':
+            return os.path.join(driverpath, 'tiles.shp')
+        raise ValueError("'{}' is not a valid setting for"
+                         " {} driver".format(key, cls.name))
 
     @classmethod
     def managed_request(cls, url, verbosity=1, debuglevel=0):