HC-213: increase backoff (#16)

* increase backoff and utilize random jitter to prevent excessive API calls

* utilize backup method for returning content_length

osaka/__init__.py

@@ -3,6 +3,6 @@
 from __future__ import division
 from __future__ import absolute_import
 
-__version__ = "1.0.0"
+__version__ = "1.0.1"
 __url__ = "https://github.com/hysds/osaka"
 __description__ = "Osaka (Object Store Abstraction K Arcitecture)"

osaka/storage/s3.py

@@ -9,6 +9,7 @@
 
 standard_library.install_aliases()
 import re
+import traceback
 import backoff
 import boto3
 import botocore
@@ -121,7 +122,7 @@ def getSchemes():
         return ["s3", "s3s"]
 
     @backoff.on_exception(
-        backoff.expo, Exception, max_value=3, max_time=13,
+        backoff.expo, Exception, factor=4, max_time=32, jitter=backoff.random_jitter
     )
     def reload_obj(self, obj):
         """
@@ -176,8 +177,17 @@ def put(self, stream, uri):
             extra = {"ServerSideEncryption": self.encrypt}
         with osaka.storage.file.FileHandlerConversion(stream) as fn:
             obj.upload_file(fn, ExtraArgs=extra)
-        self.reload_obj(obj)
-        return obj.content_length
+        try:
+            self.reload_obj(obj)
+            return obj.content_length
+        except:
+            osaka.utils.LOGGER.warn(
+                "Exponential backoff on getting content length for {} failed. {}.".format(
+                    uri, traceback.format_exc()
+                )
+            )
+            osaka.utils.LOGGER.warn("Trying alternate method.")
+            return self.size(uri)
 
     def listAllChildren(self, uri):
         """