add AWS S3 storage system

README.md

@@ -103,6 +103,14 @@ this is an improved implementation so we can maintain backwards compatibility wi
 running any migration scripts. It is also the simplest backend and it is used by
 default.
 
+### aws_s3
+
+This is a backend based on the Amazon AWS S3 storage system. The backend is activated 
+by setting TP_BACKEND=aws_s3. Each paste is stored as a separate Amazon S3 object and
+has data, a key, and metadata. The key (paste_id) uniquely identifies the object
+(paste) in a bucket. Object metadata is a set of name-value pairs that cannot be 
+modified but can be replaced by a metadata copy.
+
 ## Configuration
 TorPaste can be configured by using `ENV`ironment Variables. The list of available
 variables as well as their actions is below so you can use them to parameterize your
@@ -139,3 +147,17 @@ such as `MYSQL` and the `VARIABLE` will be the name of the variable, such as `HO
 
 Currently there are no used backend `ENV` variables. When there are, you will find
 a list of all backends and their variables here.
+
+### aws_s3
+
+This backend assumes that you have an Amazon S3 subscription and a storage account
+in that subscription. You can learn how to set up a new subscription and how to
+set up a storage account [here](http://docs.aws.amazon.com/AmazonS3/latest/gsg/SigningUpforS3.html).
+
+TP_BACKEND_AWS_S3_ACCESS_KEY_ID : Use this variable to set the key id of the
+Amazon AWS S3 account to use.
+TP_BACKEND_AWS_S3_SECRET_ACCESS_KEY : Use this variable to set the secret key
+of the Amazon AWS S3 account to use.
+TP_BACKEND_AWS_S3_BUCKET : Use this variable to set the name of the container
+in which to store pastes and metadata. If the container does not exist, it will
+be created. Default: torpaste.

backends/aws_s3.py

@@ -0,0 +1,139 @@
+from functools import wraps
+from os import environ
+from os import getenv
+
+import boto3
+from botocore.exceptions import ClientError
+
+from backends.exceptions import ErrorException
+
+_ENV_ACCESS_KEY_ID = 'TP_BACKEND_AWS_S3_ACCESS_KEY_ID'
+_ENV_SECRET_ACCESS_KEY = 'TP_BACKEND_AWS_S3_SECRET_ACCESS_KEY'
+_ENV_BUCKET = 'TP_BACKEND_AWS_S3_BUCKET'
+
+_DEFAULT_BUCKET = 'torpaste'
+
+_s3 = None
+_bucket = None
+
+
+def _wrap_aws_exception(func):
+    @wraps(func)
+    def _adapt_exception_types(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ClientError as ex:
+            raise ErrorException(
+                'Error while communicating with AWS S3') from ex
+
+    return _adapt_exception_types
+
+
+def _getenv_required(key):
+    try:
+        return environ[key]
+    except KeyError:
+        raise ErrorException(
+            'Required environment variable %s not set' % key)
+
+
+def _getenv_int(key, default):
+    try:
+        value = environ[key]
+    except KeyError:
+        return default
+
+    try:
+        return int(value)
+    except ValueError:
+        raise ErrorException(
+            'Environment variable %s with value %s '
+            'is not convertible to int' % (key, value))
+
+
+@_wrap_aws_exception
+def initialize_backend():
+    global _s3
+    global _bucket
+
+    _s3 = boto3.resource(
+        's3',
+        aws_access_key_id=_getenv_required(_ENV_ACCESS_KEY_ID),
+        aws_secret_access_key=_getenv_required(_ENV_SECRET_ACCESS_KEY))
+
+    _bucket = getenv(_ENV_BUCKET, _DEFAULT_BUCKET)
+    _s3.create_bucket(Bucket=_bucket)
+
+
+@_wrap_aws_exception
+def new_paste(paste_id, paste_content):
+    _s3.Bucket(_bucket).put_object(
+        Body=paste_content.encode('utf-8'),
+        Key=paste_id)
+
+
+@_wrap_aws_exception
+def update_paste_metadata(paste_id, metadata):
+    obj = _s3.Object(_bucket, paste_id)
+    obj.metadata.clear()
+    obj.metadata.update(metadata)
+    obj.copy_from(CopySource={'Bucket': _bucket, 'Key': paste_id},
+                  Metadata=obj.metadata,
+                  MetadataDirective='REPLACE')
+
+
+@_wrap_aws_exception
+def does_paste_exist(paste_id):
+    try:
+        _s3.Object(_bucket, paste_id).load()
+    except ClientError as ex:
+        if ex.response['Error']['Code'] != '404':
+            raise
+        else:
+            return False
+    else:
+        return True
+
+
+@_wrap_aws_exception
+def get_paste_contents(paste_id):
+    body = _s3.Object(_bucket, paste_id).get()['Body'].read()
+    return body.decode('utf-8')
+
+
+@_wrap_aws_exception
+def get_paste_metadata(paste_id):
+    obj = _s3.Object(_bucket, paste_id)
+    obj.load()
+    return obj.metadata
+
+
+@_wrap_aws_exception
+def get_paste_metadata_value(paste_id, key):
+    return get_paste_metadata(paste_id).get(key)
+
+
+def _filters_match(metadata, filters, fdefaults):
+    for metadata_key, filter_value in filters.items():
+        try:
+            metadata_value = metadata[metadata_key]
+        except KeyError:
+            metadata_value = fdefaults.get(metadata_key)
+
+        if metadata_value != filter_value:
+            return False
+
+    return True
+
+
+def _get_all_paste_ids(filters, fdefaults):
+    for obj in _s3.Bucket(_bucket).objects.all():
+        paste_id = obj.key
+        metadata = get_paste_metadata(paste_id)
+        if _filters_match(metadata, filters, fdefaults):
+            yield paste_id
+
+
+@_wrap_aws_exception
+def get_all_paste_ids(filters={}, fdefaults={}):
+    return list(_get_all_paste_ids(filters, fdefaults))

requirements.txt

@@ -1 +1,3 @@
 Flask
+boto3==1.4.7
+

torpaste.py

@@ -21,7 +21,7 @@
 VERSION = check_output(["git", "describe"]).decode("utf-8").replace("\n", "")
 
 # Compatible Backends List
-COMPATIBLE_BACKENDS = ["filesystem"]
+COMPATIBLE_BACKENDS = ["filesystem", "aws_s3"]
 
 # Available list of paste visibilities
 # public: can be viewed by all, is listed in /list