Sdk 292: Adding support to connect to s3 buckets in region with AWS S…

…ignature V4. (#237) boto version upgrade and support for downloading from AWS Signature V4 enabled buckets.
qubole · Sep 27, 2018 · 8371ea8 · 8371ea8
1 parent 1e8194e
commit 8371ea8
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py
@@ -248,9 +248,15 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N
         else:
             if fetch:
                 storage_credentials = conn.get(Account.credentials_rest_entity_path)
-                boto_conn = boto.connect_s3(aws_access_key_id=storage_credentials['storage_access_key'],
-                                            aws_secret_access_key=storage_credentials['storage_secret_key'],
-                                            security_token = storage_credentials['session_token'])
+                if storage_credentials['region_endpoint'] is not None:
+                    boto_conn = boto.connect_s3(aws_access_key_id=storage_credentials['storage_access_key'],
+                                                aws_secret_access_key=storage_credentials['storage_secret_key'],
+                                                security_token = storage_credentials['session_token'],
+                                                host = storage_credentials['region_endpoint'])
+                else:
+                    boto_conn = boto.connect_s3(aws_access_key_id=storage_credentials['storage_access_key'],
+                                                aws_secret_access_key=storage_credentials['storage_secret_key'],
+                                                security_token=storage_credentials['session_token'])
 
                 log.info("Starting download from result locations: [%s]" % ",".join(r['result_location']))
                 #fetch latest value of num_result_dir

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 import sys
 from setuptools import setup
 
-INSTALL_REQUIRES = ['requests >=1.0.3', 'boto >=2.1.1', 'six >=1.2.0', 'urllib3 >= 1.0.2', 'inflection >= 0.3.1']
+INSTALL_REQUIRES = ['requests >=1.0.3', 'boto >=2.45.0', 'six >=1.2.0', 'urllib3 >= 1.0.2', 'inflection >= 0.3.1']
 if sys.version_info < (2, 7, 0):
     INSTALL_REQUIRES.append('argparse>=1.1')