diff --git a/README.md b/README.md index 0a4d7d4..78a5309 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,14 @@ this is an improved implementation so we can maintain backwards compatibility wi running any migration scripts. It is also the simplest backend and it is used by default. +### aws_s3 + +This is a backend based on the Amazon AWS S3 storage system. The backend is activated +by setting TP_BACKEND=aws_s3. Each paste is stored as a separate Amazon S3 object and +has data, a key, and metadata. The key (paste_id) uniquely identifies the object +(paste) in a bucket. Object metadata is a set of name-value pairs that cannot be +modified but can be replaced by a metadata copy. + ## Configuration TorPaste can be configured by using `ENV`ironment Variables. The list of available variables as well as their actions is below so you can use them to parameterize your @@ -139,3 +147,17 @@ such as `MYSQL` and the `VARIABLE` will be the name of the variable, such as `HO Currently there are no used backend `ENV` variables. When there are, you will find a list of all backends and their variables here. + +### aws_s3 + +This backend assumes that you have an Amazon S3 subscription and a storage account +in that subscription. You can learn how to set up a new subscription and how to +set up a storage account [here](http://docs.aws.amazon.com/AmazonS3/latest/gsg/SigningUpforS3.html). + +TP_BACKEND_AWS_S3_ACCESS_KEY_ID : Use this variable to set the key id of the +Amazon AWS S3 account to use. +TP_BACKEND_AWS_S3_SECRET_ACCESS_KEY : Use this variable to set the secret key +of the Amazon AWS S3 account to use. +TP_BACKEND_AWS_S3_BUCKET : Use this variable to set the name of the container +in which to store pastes and metadata. If the container does not exist, it will +be created. Default: torpaste. \ No newline at end of file diff --git a/backends/aws_s3.py b/backends/aws_s3.py new file mode 100644 index 0000000..65b3c95 --- /dev/null +++ b/backends/aws_s3.py @@ -0,0 +1,139 @@ +from functools import wraps +from os import environ +from os import getenv + +import boto3 +from botocore.exceptions import ClientError + +from backends.exceptions import ErrorException + +_ENV_ACCESS_KEY_ID = 'TP_BACKEND_AWS_S3_ACCESS_KEY_ID' +_ENV_SECRET_ACCESS_KEY = 'TP_BACKEND_AWS_S3_SECRET_ACCESS_KEY' +_ENV_BUCKET = 'TP_BACKEND_AWS_S3_BUCKET' + +_DEFAULT_BUCKET = 'torpaste' + +_s3 = None +_bucket = None + + +def _wrap_aws_exception(func): + @wraps(func) + def _adapt_exception_types(*args, **kwargs): + try: + return func(*args, **kwargs) + except ClientError as ex: + raise ErrorException( + 'Error while communicating with AWS S3') from ex + + return _adapt_exception_types + + +def _getenv_required(key): + try: + return environ[key] + except KeyError: + raise ErrorException( + 'Required environment variable %s not set' % key) + + +def _getenv_int(key, default): + try: + value = environ[key] + except KeyError: + return default + + try: + return int(value) + except ValueError: + raise ErrorException( + 'Environment variable %s with value %s ' + 'is not convertible to int' % (key, value)) + + +@_wrap_aws_exception +def initialize_backend(): + global _s3 + global _bucket + + _s3 = boto3.resource( + 's3', + aws_access_key_id=_getenv_required(_ENV_ACCESS_KEY_ID), + aws_secret_access_key=_getenv_required(_ENV_SECRET_ACCESS_KEY)) + + _bucket = getenv(_ENV_BUCKET, _DEFAULT_BUCKET) + _s3.create_bucket(Bucket=_bucket) + + +@_wrap_aws_exception +def new_paste(paste_id, paste_content): + _s3.Bucket(_bucket).put_object( + Body=paste_content.encode('utf-8'), + Key=paste_id) + + +@_wrap_aws_exception +def update_paste_metadata(paste_id, metadata): + obj = _s3.Object(_bucket, paste_id) + obj.metadata.clear() + obj.metadata.update(metadata) + obj.copy_from(CopySource={'Bucket': _bucket, 'Key': paste_id}, + Metadata=obj.metadata, + MetadataDirective='REPLACE') + + +@_wrap_aws_exception +def does_paste_exist(paste_id): + try: + _s3.Object(_bucket, paste_id).load() + except ClientError as ex: + if ex.response['Error']['Code'] != '404': + raise + else: + return False + else: + return True + + +@_wrap_aws_exception +def get_paste_contents(paste_id): + body = _s3.Object(_bucket, paste_id).get()['Body'].read() + return body.decode('utf-8') + + +@_wrap_aws_exception +def get_paste_metadata(paste_id): + obj = _s3.Object(_bucket, paste_id) + obj.load() + return obj.metadata + + +@_wrap_aws_exception +def get_paste_metadata_value(paste_id, key): + return get_paste_metadata(paste_id).get(key) + + +def _filters_match(metadata, filters, fdefaults): + for metadata_key, filter_value in filters.items(): + try: + metadata_value = metadata[metadata_key] + except KeyError: + metadata_value = fdefaults.get(metadata_key) + + if metadata_value != filter_value: + return False + + return True + + +def _get_all_paste_ids(filters, fdefaults): + for obj in _s3.Bucket(_bucket).objects.all(): + paste_id = obj.key + metadata = get_paste_metadata(paste_id) + if _filters_match(metadata, filters, fdefaults): + yield paste_id + + +@_wrap_aws_exception +def get_all_paste_ids(filters={}, fdefaults={}): + return list(_get_all_paste_ids(filters, fdefaults)) diff --git a/requirements.txt b/requirements.txt index e3e9a71..b7cf435 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ Flask +boto3==1.4.7 + diff --git a/torpaste.py b/torpaste.py index ab953cd..690f9fe 100755 --- a/torpaste.py +++ b/torpaste.py @@ -21,7 +21,7 @@ VERSION = check_output(["git", "describe"]).decode("utf-8").replace("\n", "") # Compatible Backends List -COMPATIBLE_BACKENDS = ["filesystem"] +COMPATIBLE_BACKENDS = ["filesystem", "aws_s3"] # Available list of paste visibilities # public: can be viewed by all, is listed in /list