From 0c2a049276189b017b7e95eb4f6d5c038712ebd1 Mon Sep 17 00:00:00 2001 From: Alex Schoof Date: Wed, 4 Nov 2015 08:47:24 -0500 Subject: [PATCH 1/6] pad auto-generated version numbers --- credstash.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/credstash.py b/credstash.py index 85e8312..79045f9 100755 --- a/credstash.py +++ b/credstash.py @@ -44,6 +44,7 @@ from Crypto.Util import Counter DEFAULT_REGION = "us-east-1" +PAD_LEN = 19 # number of digits in sys.maxint WILDCARD_CHAR = "*" @@ -124,6 +125,14 @@ def csv_dump(dictionary): return csvfile.getvalue() +def paddedInt(i): + ''' + return a string that contains `i`, left-padded with 0's up to PAD_LEN digits + ''' + i_str = str(i) + pad = PAD_LEN - len(i_str) + return (pad * "0") + i_str + def getHighestVersion(name, region="us-east-1", table="credential-store"): ''' Return the highest version of `name` in the table @@ -187,7 +196,7 @@ def putSecret(name, secret, version, kms_key="alias/credstash", data = {} data['name'] = name - data['version'] = version if version != "" else "1" + data['version'] = version if version != "" else paddedInt(1) data['key'] = b64encode(wrapped_key).decode('utf-8') data['contents'] = b64encode(c_text).decode('utf-8') data['hmac'] = b64hmac @@ -470,7 +479,7 @@ def main(): latestVersion = getHighestVersion(args.credential, region, args.table) try: - version = str(int(latestVersion) + 1) + version = paddedInt(int(latestVersion) + 1) except ValueError: printStdErr("Can not autoincrement version. The current " "version: %s is not an int" % latestVersion) From 6ed776684cdd0133ea89b6a36c8c51de7574bd96 Mon Sep 17 00:00:00 2001 From: dom-at-luminal Date: Thu, 5 Nov 2015 21:02:19 -0500 Subject: [PATCH 2/6] first cut at migration script --- credstash-migrate-autoversion.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 credstash-migrate-autoversion.py diff --git a/credstash-migrate-autoversion.py b/credstash-migrate-autoversion.py new file mode 100644 index 0000000..983d780 --- /dev/null +++ b/credstash-migrate-autoversion.py @@ -0,0 +1,32 @@ +import boto3 +import credstash +import copy + + +def updateVersions(region="us-east-1", table="credential-store"): + ''' + do a full-table scan of the credential-store, + and update the version format of every credential if it is an integer + ''' + dynamodb = boto3.resource('dynamodb', region_name=region) + secrets = dynamodb.Table(table) + + response = secrets.scan(ProjectionExpression="#N, version, #K, contents, hmac", + ExpressionAttributeNames={"#N": "name", "#K": "key"}) + + items = response["Items"] + + for old_item in items: + try: + int(old_item['version']) + new_item = copy.copy(old_item) + new_item['version'] = credstash.paddedInt(new_item['version']) + if new_item['version'] != old_item['version']: + secrets.put_item(Item=new_item) + secrets.delete_item(Key={'name': old_item['name'], 'version': old_item['version']}) + except: + print "Skipping item: %s, %s" % (old_item['name'], old_item['version']) + + +if __name__ == "__main__": + updateVersions() From 1eb4d83b0f6c41c0ecf2d6de3ef45aa91bda2c35 Mon Sep 17 00:00:00 2001 From: dom-at-luminal Date: Fri, 6 Nov 2015 07:31:03 -0500 Subject: [PATCH 3/6] moving the terrible flow control to an isolated function --- credstash-migrate-autoversion.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/credstash-migrate-autoversion.py b/credstash-migrate-autoversion.py index 983d780..7e5c706 100644 --- a/credstash-migrate-autoversion.py +++ b/credstash-migrate-autoversion.py @@ -3,6 +3,14 @@ import copy +def isInt(s): + try: + int(s) + return True + except ValueError: + return False + + def updateVersions(region="us-east-1", table="credential-store"): ''' do a full-table scan of the credential-store, @@ -17,14 +25,13 @@ def updateVersions(region="us-east-1", table="credential-store"): items = response["Items"] for old_item in items: - try: - int(old_item['version']) + if isInt(old_item['version']): new_item = copy.copy(old_item) new_item['version'] = credstash.paddedInt(new_item['version']) if new_item['version'] != old_item['version']: secrets.put_item(Item=new_item) secrets.delete_item(Key={'name': old_item['name'], 'version': old_item['version']}) - except: + else: print "Skipping item: %s, %s" % (old_item['name'], old_item['version']) From 28735715e89d0432f3585b00b3d8833591516536 Mon Sep 17 00:00:00 2001 From: dom-at-luminal Date: Wed, 2 Dec 2015 09:28:24 -0500 Subject: [PATCH 4/6] README updates --- README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5f1443f..a904e88 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,24 @@ CredStash is a very simple, easy to use credential management and distribution s ## How does it work? After you complete the steps in the `Setup` section, you will have an encryption key in KMS (in this README, we will refer to that key as the `master key`), and a credential storage table in DDB. +### Stashing Secrets Whenever you want to store/share a credential, such as a database password, you simply run `credstash put [credential-name] [credential-value]`. For example, `credstash put myapp.db.prod supersecretpassword1234`. credstash will go to the KMS and generate a unique data encryption key, which itself is encrypted by the master key (this is called key wrapping). credstash will use the data encryption key to encrypt the credential value. It will then store the encrypted credential, along with the wrapped (encrypted) data encryption key in the credential store in DynamoDB. +### Getting Secrets When you want to fetch the credential, for example as part of the bootstrap process on your web-server, you simply do `credstash get [credential-name]`. For example, `export DB_PASSWORD=$(credstash get myapp.db.prod)`. When you run `get`, credstash will go and fetch the encrypted credential and the wrapped encryption key from the credential store (DynamoDB). It will then send the wrapped encryption key to KMS, where it is decrypted with the master key. credstash then uses the decrypted data encryption key to decrypt the credential. The credential is printed to `stdout`, so you can use it in scripts or assign environment variables to it. -Optionally you can include any number of [Encryption Context](http://docs.aws.amazon.com/kms/latest/developerguide/encrypt-context.html) key value pairs to associate with the credential. The exact set of encryption context key value pairs that were associated with the credential when it was `put` in DynamoDB must be provided in the `get` request to successfully decrypt the credential. These encryption context key value pairs are useful to provide auditing context to the encryption and decryption operations in your CloudTrail logs. They are also useful for constraining access to a given credstash stored credential by using KMS Key Policy conditions and KMS Grant conditions. Doing so allows you to, for example, make sure that your database servers and web-servers can read the web-server DB user password but your database servers can not read your web-servers TLS/SSL certificate's private key. A `put` request with encryption context would look like `credstash put myapp.db.prod supersecretpassword1234 app.tier=db environment=prod`. In order for your web-servers to read that same credential they would execute a `get` call like `export DB_PASSWORD=$(credstash get myapp.db.prod environment=prod app.tier=db)` +### Controlling and Auditing Secrets +Optionally, you can include any number of [Encryption Context](http://docs.aws.amazon.com/kms/latest/developerguide/encrypt-context.html) key value pairs to associate with the credential. The exact set of encryption context key value pairs that were associated with the credential when it was `put` in DynamoDB must be provided in the `get` request to successfully decrypt the credential. These encryption context key value pairs are useful to provide auditing context to the encryption and decryption operations in your CloudTrail logs. They are also useful for constraining access to a given credstash stored credential by using KMS Key Policy conditions and KMS Grant conditions. Doing so allows you to, for example, make sure that your database servers and web-servers can read the web-server DB user password but your database servers can not read your web-servers TLS/SSL certificate's private key. A `put` request with encryption context would look like `credstash put myapp.db.prod supersecretpassword1234 app.tier=db environment=prod`. In order for your web-servers to read that same credential they would execute a `get` call like `export DB_PASSWORD=$(credstash get myapp.db.prod environment=prod app.tier=db)` -Credentials stored in the credential-store are versioned and immutable. That is, if you `put` a credential called `foo` with a version of `1` and a value of `bar`, then foo version 1 will always have a value of bar, and there is no way in `credstash` to change its value (although you could go fiddle with the bits in DDB, but you shouldn't do that). Credential rotation is handed through versions. Suppose you do `credstash put foo bar`, and then decide later to rotate `foo`, you can put version 2 of `foo` by doing `credstash put foo baz -v `. The next time you do `credstash get foo`, it will return `baz`. You can get specific credential versions as well (with the same `-v` flag). You can fetch a list of all credentials in the credential-store and their versions with the `list` command. If you use integer version numbers (1,2,3,...), then you can use the `-a` flag with the `put` command to automatically increment the version number. +### Versioning Secrets +Credentials stored in the credential-store are versioned and immutable. That is, if you `put` a credential called `foo` with a version of `1` and a value of `bar`, then foo version 1 will always have a value of bar, and there is no way in `credstash` to change its value (although you could go fiddle with the bits in DDB, but you shouldn't do that). Credential rotation is handed through versions. Suppose you do `credstash put foo bar`, and then decide later to rotate `foo`, you can put version 2 of `foo` by doing `credstash put foo baz -v `. The next time you do `credstash get foo`, it will return `baz`. You can get specific credential versions as well (with the same `-v` flag). You can fetch a list of all credentials in the credential-store and their versions with the `list` command. + +If you use incrementing integer version numbers (for example, `[1, 2, 3, ...]`), then you can use the `-a` flag with the `put` command to automatically increment the version number. However, because of the lexicographical sorting in DynamoDB, `credstash` will left-pad the version representation with zeros (for example, `[001, 025, 103, ...]`, except to 19 characters, enough to handle `sys.maxint` on 64-bit systems). + +#### Special Note for Those Using Credstash Auto-Versioning Before December 2015 +Prior to December 2015, `credstash` auto-versioned with unpadded integers. This resulted in a sorting error once a key hit ten versions. To ensure support for dates that were not numbers (such as dates, build versions, names, etc.), the lexicographical sorting behavior was retained, but the auto-versioning behavior was changed to left-pad integer representations. + +If you've used auto-versioning so far, you should run the `credstash-migrate-autoversion.py` script included in the root of the repository. If you are supplying your own version numbers, you should ensure a lexicographic sort of your versions produces the result you desire. ## Dependencies credstash uses the following AWS services: @@ -97,11 +108,11 @@ get getall usage: credstash getall [-h] [-r REGION] [-t TABLE] [-v VERSION] [-f {json,yaml,csv}] [context [context ...]] - + positional arguments: context encryption context key/value pairs associated with the credential in the form of "key=value" - + optional arguments: -v VERSION, --version VERSION Get a specific version of the credential (defaults to @@ -176,7 +187,7 @@ You can put or write secrets to credstash by either using KMS Key Grants, KMS Ke ] } ``` -If you are using Key Policies or Grants, then the `kms:GenerateDataKey` is not required in the policy for the IAM user/group/role. Replace `AWSACCOUNTID` with the account ID for your table, and replace the KEY-GUID with the identifier for your KMS key (which you can find in the KMS console). +If you are using Key Policies or Grants, then the `kms:GenerateDataKey` is not required in the policy for the IAM user/group/role. Replace `AWSACCOUNTID` with the account ID for your table, and replace the KEY-GUID with the identifier for your KMS key (which you can find in the KMS console). ### Secret Reader You can read secrets from credstash with the get or getall actions by either using KMS Key Grants, KMS Key Policies, or IAM Policies. If you are using IAM Policies, the following IAM permissions are the minimum required to be able to put or read secrets: From 44c176b2ad0c4844c7e27af8bbf4a4a5c11392cb Mon Sep 17 00:00:00 2001 From: Alex Schoof Date: Wed, 2 Dec 2015 09:56:16 -0500 Subject: [PATCH 5/6] added shebang to migration script --- credstash-migrate-autoversion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/credstash-migrate-autoversion.py b/credstash-migrate-autoversion.py index 7e5c706..020541c 100644 --- a/credstash-migrate-autoversion.py +++ b/credstash-migrate-autoversion.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import boto3 import credstash import copy From 62c98b13af42b9ee31918d8adc887f42b1d24061 Mon Sep 17 00:00:00 2001 From: Alex Schoof Date: Wed, 2 Dec 2015 09:56:27 -0500 Subject: [PATCH 6/6] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 842f4b7..0697f09 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='credstash', - version='1.7', + version='1.8', description='A utility for managing secrets in the cloud using AWS KMS and DynamoDB', license='Apache2', url='https://github.com/LuminalOSS/credstash',